From 382b73d99ec8ab69bf5dc6ec231e237fadeca30d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 10 Apr 2024 11:17:59 -0400
Subject: [PATCH 001/220] fix nuclei budget bug

---
 bbot/modules/deadly/nuclei.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py
index 04be1aa95..2e40b4bfd 100644
--- a/bbot/modules/deadly/nuclei.py
+++ b/bbot/modules/deadly/nuclei.py
@@ -322,13 +322,15 @@ def get_yaml_request_attr(self, yf, attr):
             raw = r.get("raw")
             if not raw:
                 res = r.get(attr)
-                yield res
+                if res is not None:
+                    yield res
 
     def get_yaml_info_attr(self, yf, attr):
         p = self.parse_yaml(yf)
         info = p.get("info", [])
         res = info.get(attr)
-        yield res
+        if res is not None:
+            yield res
 
     # Parse through all templates and locate those which match the conditions necessary to collapse down to the budget setting
     def find_collapsible_templates(self):

From b037471586ec026a3d643354b9dad9b3efd89cd1 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 23 Apr 2024 12:24:56 -0400
Subject: [PATCH 002/220] remove resolved/unresolved tags as they are redundant

---
 bbot/modules/anubisdb.py          | 2 +-
 bbot/modules/internal/dns.py      | 4 +---
 bbot/test/test_step_1/test_dns.py | 3 +--
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py
index 9864e3c6d..bf4c88e93 100644
--- a/bbot/modules/anubisdb.py
+++ b/bbot/modules/anubisdb.py
@@ -30,7 +30,7 @@ def abort_if_pre(self, hostname):
 
     async def abort_if(self, event):
         # abort if dns name is unresolved
-        if not "resolved" in event.tags:
+        if event.type == "DNS_NAME_UNRESOLVED":
             return True, "DNS name is unresolved"
         return await super().abort_if(event)
 
diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index ea5e4efcf..b96b9b19c 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -94,9 +94,7 @@ async def handle_event(self, event, kwargs):
                     if rdtype not in dns_children:
                         dns_tags.add(f"{rdtype.lower()}-error")
 
-                if dns_children:
-                    dns_tags.add("resolved")
-                elif not event_is_ip:
+                if not dns_children and not event_is_ip:
                     dns_tags.add("unresolved")
 
                 for rdtype, children in dns_children.items():
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 05796e464..aa2a27907 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -238,7 +238,6 @@ async def test_wildcards(bbot_scanner):
                     "a-record",
                     "target",
                     "aaaa-wildcard",
-                    "resolved",
                     "in-scope",
                     "subdomain",
                     "aaaa-record",
@@ -260,7 +259,7 @@ async def test_wildcards(bbot_scanner):
             for e in events
             if e.type == "DNS_NAME"
             and e.data == "asdfl.gashdgkjsadgsdf.github.io"
-            and all(t in e.tags for t in ("a-record", "target", "resolved", "in-scope", "subdomain", "aaaa-record"))
+            and all(t in e.tags for t in ("a-record", "target", "in-scope", "subdomain", "aaaa-record"))
             and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard"))
         ]
     )

From 37a5889e7f3c97b4d5829f9dcfe0e7a3d6bc15fa Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 19 Apr 2024 13:52:50 -0400
Subject: [PATCH 003/220] tests for custom target types

---
 bbot/test/test_step_1/test_cli.py        | 8 ++++++++
 bbot/test/test_step_1/test_python_api.py | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index db2a5316d..d879d863d 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -190,6 +190,14 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     assert "| active" in caplog.text
     assert not "| passive" in caplog.text
 
+    # custom target type
+    caplog.clear()
+    assert not caplog.text
+    monkeypatch.setattr("sys.argv", ["bbot", "-t", "ORG:evilcorp"])
+    result = await cli._main()
+    assert result == True
+    assert "[ORG_STUB]          	evilcorp	TARGET" in caplog.text
+
     # no args
     caplog.clear()
     assert not caplog.text
diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py
index 14c2fae1d..0155dcfb3 100644
--- a/bbot/test/test_step_1/test_python_api.py
+++ b/bbot/test/test_step_1/test_python_api.py
@@ -45,6 +45,11 @@ async def test_python_api():
     Scanner("127.0.0.1", config={"home": bbot_home})
     assert os.environ["BBOT_TOOLS"] == str(Path(bbot_home) / "tools")
 
+    # custom target types
+    custom_target_scan = Scanner("ORG:evilcorp")
+    events = [e async for e in custom_target_scan.async_start()]
+    assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "evilcorp" and "target" in e.tags])
+
 
 def test_python_api_sync():
     from bbot.scanner import Scanner

From b06355bc1cbbd8714786d11b08f66b59b11d583d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 23 Apr 2024 12:09:10 -0400
Subject: [PATCH 004/220] fix small cli bug and add tests for it

---
 bbot/scanner/preset/preset.py     | 6 ++++--
 bbot/test/test_step_1/test_cli.py | 7 +++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 60dac85dd..57eb11a1a 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -381,8 +381,10 @@ def bake(self):
         for flag in baked_preset.flags:
             for module, preloaded in baked_preset.module_loader.preloaded().items():
                 module_flags = preloaded.get("flags", [])
+                module_type = preloaded.get("type", "scan")
                 if flag in module_flags:
-                    baked_preset.add_module(module, raise_error=False)
+                    self.log_debug(f'Enabling module "{module}" because it has flag "{flag}"')
+                    baked_preset.add_module(module, module_type, raise_error=False)
 
         # ensure we have output modules
         if not baked_preset.output_modules:
@@ -433,7 +435,7 @@ def internal_modules(self):
         return [m for m in self.modules if self.preloaded_module(m).get("type", "scan") == "internal"]
 
     def add_module(self, module_name, module_type="scan", raise_error=True):
-        self.log_debug(f'Adding module "{module_name}"')
+        self.log_debug(f'Adding module "{module_name}" of type "{module_type}"')
         is_valid, reason, preloaded = self._is_valid_module(module_name, module_type, raise_error=raise_error)
         if not is_valid:
             self.log_debug(f'Unable to add {module_type} module "{module_name}": {reason}')
diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index d879d863d..0a34b4faa 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -198,6 +198,13 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     assert result == True
     assert "[ORG_STUB]          	evilcorp	TARGET" in caplog.text
 
+    # activate modules by flag
+    caplog.clear()
+    assert not caplog.text
+    monkeypatch.setattr("sys.argv", ["bbot", "-f", "passive"])
+    result = await cli._main()
+    assert result == True
+
     # no args
     caplog.clear()
     assert not caplog.text

From 84df8298e49f6f7e68723829ddece6a70c43e71e Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 17 Apr 2024 16:38:24 -0400
Subject: [PATCH 005/220] fix attribute error

---
 bbot/core/helpers/command.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index 14a788f8a..06fc8a91f 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -185,7 +185,8 @@ async def _write_proc_line(proc, chunk):
         proc.stdin.write(smart_encode(chunk) + b"\n")
         await proc.stdin.drain()
     except Exception as e:
-        command = " ".join([str(s) for s in proc.args])
+        proc_args = [str(s) for s in getattr(proc, "args", [])]
+        command = " ".join(proc_args)
         log.warning(f"Error writing line to stdin for command: {command}: {e}")
         log.trace(traceback.format_exc())
 

From 355a5bee6ee60b4c664ab25a56e5be0dc1f80e38 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 23 Apr 2024 11:50:21 -0400
Subject: [PATCH 006/220] Better debugging during scan cancellation

---
 bbot/core/helpers/misc.py |  2 ++
 bbot/scanner/scanner.py   | 14 ++++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index a4378069d..206fc50f0 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -1078,6 +1078,7 @@ def kill_children(parent_pid=None, sig=None):
         parent = psutil.Process(parent_pid)
     except psutil.NoSuchProcess:
         log.debug(f"No such PID: {parent_pid}")
+        return
     log.debug(f"Killing children of process ID {parent.pid}")
     children = parent.children(recursive=True)
     for child in children:
@@ -1089,6 +1090,7 @@ def kill_children(parent_pid=None, sig=None):
                 log.debug(f"No such PID: {child.pid}")
             except psutil.AccessDenied:
                 log.debug(f"Error killing PID: {child.pid} - access denied")
+    log.debug(f"Finished killing children of process ID {parent.pid}")
 
 
 def str_or_file(s):
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index e01f85ce2..9f444d3cb 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -351,7 +351,13 @@ async def async_start(self):
                     self.critical(f"Unexpected error during scan:\n{traceback.format_exc()}")
 
         finally:
-            self._cancel_tasks()
+            tasks = self._cancel_tasks()
+            self.debug(f"Awaiting {len(tasks):,} tasks")
+            for task in tasks:
+                self.debug(f"Awaiting {task}")
+                with contextlib.suppress(BaseException):
+                    await task
+            self.debug(f"Awaited {len(tasks):,} tasks")
             await self._report()
             await self._cleanup()
 
@@ -663,13 +669,14 @@ def stop(self):
         if not self._stopping:
             self._stopping = True
             self.status = "ABORTING"
-            self.hugewarning(f"Aborting scan")
+            self.hugewarning("Aborting scan")
             self.trace()
             self._cancel_tasks()
             self._drain_queues()
             self.helpers.kill_children()
             self._drain_queues()
             self.helpers.kill_children()
+            self.debug("Finished aborting scan")
 
     async def finish(self):
         """Finalizes the scan by invoking the `finished()` method on all active modules if new activity is detected.
@@ -729,6 +736,7 @@ def _cancel_tasks(self):
         Returns:
             None
         """
+        self.debug("Cancelling all scan tasks")
         tasks = []
         # module workers
         for m in self.modules.values():
@@ -746,6 +754,8 @@ def _cancel_tasks(self):
         self.helpers.cancel_tasks_sync(tasks)
         # process pool
         self.helpers.process_pool.shutdown(cancel_futures=True)
+        self.debug("Finished cancelling all scan tasks")
+        return tasks
 
     async def _report(self):
         """Asynchronously executes the `report()` method for each module in the scan.

From 616fe2e70ea6f2cf52ea0be4cce2a5b6d50b5d32 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 23 Apr 2024 12:53:57 -0400
Subject: [PATCH 007/220] better engine error handling during scan cancellation

---
 bbot/core/engine.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index c72eecbb3..24781ab3b 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -153,19 +153,26 @@ async def run_and_return(self, client_id, command_fn, **kwargs):
             error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
             trace = traceback.format_exc()
             result = {"_e": (error, trace)}
-        await self.socket.send_multipart([client_id, pickle.dumps(result)])
+        await self.send_socket_multipart([client_id, pickle.dumps(result)])
 
     async def run_and_yield(self, client_id, command_fn, **kwargs):
         self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({kwargs})")
         try:
             async for _ in command_fn(**kwargs):
-                await self.socket.send_multipart([client_id, pickle.dumps(_)])
-            await self.socket.send_multipart([client_id, pickle.dumps({"_s": None})])
+                await self.send_socket_multipart([client_id, pickle.dumps(_)])
+            await self.send_socket_multipart([client_id, pickle.dumps({"_s": None})])
         except Exception as e:
             error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
             trace = traceback.format_exc()
             result = {"_e": (error, trace)}
-            await self.socket.send_multipart([client_id, pickle.dumps(result)])
+            await self.send_socket_multipart([client_id, pickle.dumps(result)])
+
+    async def send_socket_multipart(self, *args, **kwargs):
+        try:
+            await self.socket.send_multipart(*args, **kwargs)
+        except Exception as e:
+            self.log.warning(f"Error sending ZMQ message: {e}")
+            self.log.trace(traceback.format_exc())
 
     async def worker(self):
         try:

From 820c15dd9432e186604ec3211bd2b1b27770d356 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 24 Apr 2024 13:42:30 -0400
Subject: [PATCH 008/220] wip dnsbrute rework

---
 bbot/core/helpers/dns/dns.py                  |  11 +
 bbot/core/helpers/helper.py                   |   4 +-
 bbot/modules/dnscommonsrv.py                  |  32 +-
 bbot/modules/massdns.py                       | 417 ------------------
 bbot/modules/templates/subdomain_enum.py      |  47 +-
 bbot/scanner/preset/preset.py                 |   6 +-
 bbot/scanner/target.py                        |   6 +-
 .../module_tests/test_module_dastardly.py     |   2 +-
 .../module_tests/test_module_dnscommonsrv.py  |  45 +-
 .../module_tests/test_module_massdns.py       |  10 -
 10 files changed, 87 insertions(+), 493 deletions(-)
 delete mode 100644 bbot/modules/massdns.py
 delete mode 100644 bbot/test/test_step_2/module_tests/test_module_massdns.py

diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py
index 7f775483c..a4d626e5c 100644
--- a/bbot/core/helpers/dns/dns.py
+++ b/bbot/core/helpers/dns/dns.py
@@ -73,6 +73,9 @@ def __init__(self, parent_helper):
         # TODO: DNS server speed test (start in background task)
         self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False)
 
+        # brute force helper
+        self._brute = None
+
     async def resolve(self, query, **kwargs):
         return await self.run_and_return("resolve", query=query, **kwargs)
 
@@ -84,6 +87,14 @@ async def resolve_raw_batch(self, queries):
         async for _ in self.run_and_yield("resolve_raw_batch", queries=queries):
             yield _
 
+    @property
+    def brute(self):
+        if self._brute is None:
+            from .brute import DNSBrute
+
+            self._brute = DNSBrute(self.parent_helper)
+        return self._brute
+
     async def is_wildcard(self, query, ips=None, rdtype=None):
         """
         Use this method to check whether a *host* is a wildcard entry
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index 16afc05cd..e4dc09326 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -110,8 +110,8 @@ def clean_old_scans(self):
         _filter = lambda x: x.is_dir() and self.regexes.scan_name_regex.match(x.name)
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
-    def make_target(self, *events):
-        return Target(*events)
+    def make_target(self, *events, **kwargs):
+        return Target(*events, **kwargs)
 
     @property
     def config(self):
diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py
index eef8e2d8c..ae4f39c88 100644
--- a/bbot/modules/dnscommonsrv.py
+++ b/bbot/modules/dnscommonsrv.py
@@ -1,4 +1,4 @@
-from bbot.modules.base import BaseModule
+from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 # the following are the result of a 1-day internet survey to find the top SRV records
 # the scan resulted in 36,282 SRV records. the count for each one is shown.
@@ -149,33 +149,15 @@
 ]
 
 
-class dnscommonsrv(BaseModule):
+class dnscommonsrv(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
     meta = {"description": "Check for common SRV records"}
-    options = {"top": 50, "max_event_handlers": 10}
-    options_desc = {
-        "top": "How many of the top SRV records to check",
-        "max_event_handlers": "How many instances of the module to run concurrently",
-    }
-    _max_event_handlers = 10
-
-    def _incoming_dedup_hash(self, event):
-        # dedupe by parent
-        parent_domain = self.helpers.parent_domain(event.data)
-        return hash(parent_domain), "already processed parent domain"
-
-    async def filter_event(self, event):
-        # skip SRV wildcards
-        if "SRV" in await self.helpers.is_wildcard(event.host):
-            return False
-        return True
 
     async def handle_event(self, event):
-        top = int(self.config.get("top", 50))
-        parent_domain = self.helpers.parent_domain(event.data)
-        queries = [f"{srv}.{parent_domain}" for srv in common_srvs[:top]]
-        async for query, results in self.helpers.resolve_batch(queries, type="srv"):
-            if results:
-                await self.emit_event(query, "DNS_NAME", tags=["srv-record"], source=event)
+        self.hugesuccess(event)
+        query = self.make_query(event)
+        self.verbose(f'Brute-forcing SRV records for "{query}"')
+        for hostname in await self.helpers.dns.brute(self, query, common_srvs, type="SRV"):
+            await self.emit_event(hostname, "DNS_NAME", source=event)
diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py
deleted file mode 100644
index ffacb8c64..000000000
--- a/bbot/modules/massdns.py
+++ /dev/null
@@ -1,417 +0,0 @@
-import json
-import random
-import subprocess
-import regex as re
-
-from bbot.modules.templates.subdomain_enum import subdomain_enum
-
-
-class massdns(subdomain_enum):
-    """
-    This is BBOT's flagship subdomain enumeration module.
-
-    It uses massdns to brute-force subdomains.
-    At the end of a scan, it will leverage BBOT's word cloud to recursively discover target-specific subdomain mutations.
-
-    Each subdomain discovered via mutations is tagged with the "mutation" tag. This tag indicates the depth at which
-    the mutation was found. I.e. the first mutation will be tagged "mutation-1". The second one (a mutation of a
-    mutation) will be "mutation-2". Mutations of mutations of mutations will be "mutation-3", etc.
-
-    This is especially use for bug bounties because it enables you to recognize distant/rare subdomains at a glance.
-    Subdomains with higher mutation levels are more likely to be distant/rare or never-before-seen.
-    """
-
-    flags = ["subdomain-enum", "passive", "aggressive"]
-    watched_events = ["DNS_NAME"]
-    produced_events = ["DNS_NAME"]
-    meta = {"description": "Brute-force subdomains with massdns (highly effective)"}
-    options = {
-        "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt",
-        "max_resolvers": 1000,
-        "max_mutations": 500,
-        "max_depth": 5,
-    }
-    options_desc = {
-        "wordlist": "Subdomain wordlist URL",
-        "max_resolvers": "Number of concurrent massdns resolvers",
-        "max_mutations": "Max number of smart mutations per subdomain",
-        "max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com",
-    }
-    subdomain_file = None
-    deps_common = ["massdns"]
-    reject_wildcards = "strict"
-    _qsize = 10000
-
-    digit_regex = re.compile(r"\d+")
-
-    async def setup(self):
-        self.found = dict()
-        self.mutations_tried = set()
-        self.source_events = self.helpers.make_target()
-        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
-        self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
-
-        ms_on_prem_string_file = self.helpers.wordlist_dir / "ms_on_prem_subdomains.txt"
-        ms_on_prem_strings = set(self.helpers.read_file(ms_on_prem_string_file))
-        self.subdomain_list.update(ms_on_prem_strings)
-
-        self.max_resolvers = self.config.get("max_resolvers", 1000)
-        self.max_mutations = self.config.get("max_mutations", 500)
-        self.max_depth = max(1, self.config.get("max_depth", 5))
-        nameservers_url = (
-            "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
-        )
-        self.resolver_file = await self.helpers.wordlist(
-            nameservers_url,
-            cache_hrs=24 * 7,
-        )
-        self.devops_mutations = list(self.helpers.word_cloud.devops_mutations)
-        self._mutation_run = 1
-
-        return await super().setup()
-
-    async def filter_event(self, event):
-        query = self.make_query(event)
-        eligible, reason = await self.eligible_for_enumeration(event)
-
-        # limit brute force depth
-        subdomain_depth = self.helpers.subdomain_depth(query) + 1
-        if subdomain_depth > self.max_depth:
-            eligible = False
-            reason = f"subdomain depth of *.{query} ({subdomain_depth}) > max_depth ({self.max_depth})"
-
-        # don't brute-force things that look like autogenerated PTRs
-        if self.helpers.is_ptr(query):
-            eligible = False
-            reason = f'"{query}" looks like an autogenerated PTR'
-
-        if eligible:
-            self.add_found(event)
-        # reject if already processed
-        if self.already_processed(query):
-            return False, f'Query "{query}" was already processed'
-
-        if eligible:
-            self.processed.add(hash(query))
-            return True, reason
-        return False, reason
-
-    async def handle_event(self, event):
-        query = self.make_query(event)
-        self.source_events.add_target(event)
-        self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
-        for hostname in await self.massdns(query, self.subdomain_list):
-            await self.emit_result(hostname, event, query)
-
-    def abort_if(self, event):
-        if not event.scope_distance == 0:
-            return True, "event is not in scope"
-        if "wildcard" in event.tags:
-            return True, "event is a wildcard"
-        if "unresolved" in event.tags:
-            return True, "event is unresolved"
-        return False, ""
-
-    async def emit_result(self, result, source_event, query, tags=None):
-        if not result == source_event:
-            kwargs = {"abort_if": self.abort_if}
-            if tags is not None:
-                kwargs["tags"] = tags
-            await self.emit_event(result, "DNS_NAME", source_event, **kwargs)
-
-    def already_processed(self, hostname):
-        if hash(hostname) in self.processed:
-            return True
-        return False
-
-    async def massdns(self, domain, subdomains):
-        subdomains = list(subdomains)
-
-        domain_wildcard_rdtypes = set()
-        for _domain, rdtypes in (await self.helpers.is_wildcard_domain(domain)).items():
-            for rdtype, results in rdtypes.items():
-                if results:
-                    domain_wildcard_rdtypes.add(rdtype)
-        if any([r in domain_wildcard_rdtypes for r in ("A", "CNAME")]):
-            self.info(
-                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
-            )
-            self.found.pop(domain, None)
-            return []
-        else:
-            self.log.trace(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
-
-        # before we start, do a canary check for wildcards
-        abort_msg = f"Aborting massdns on {domain} due to false positive"
-        canary_result = await self._canary_check(domain)
-        if canary_result:
-            self.info(abort_msg + f": {canary_result}")
-            return []
-        else:
-            self.log.trace(f"Canary result for {domain}: {canary_result}")
-
-        results = []
-        async for hostname, ip, rdtype in self._massdns(domain, subdomains):
-            # allow brute-forcing of wildcard domains
-            # this is dead code but it's kinda cool so it can live here
-            if rdtype in domain_wildcard_rdtypes:
-                # skip wildcard checking on multi-level subdomains for performance reasons
-                stem = hostname.split(domain)[0].strip(".")
-                if "." in stem:
-                    self.debug(f"Skipping {hostname}:A because it may be a wildcard (reason: performance)")
-                    continue
-                wildcard_rdtypes = await self.helpers.is_wildcard(hostname, ips=(ip,), rdtype=rdtype)
-                if rdtype in wildcard_rdtypes:
-                    self.debug(f"Skipping {hostname}:{rdtype} because it's a wildcard")
-                    continue
-            results.append(hostname)
-
-        # do another canary check for good measure
-        if len(results) > 50:
-            canary_result = await self._canary_check(domain)
-            if canary_result:
-                self.info(abort_msg + f": {canary_result}")
-                return []
-            else:
-                self.log.trace(f"Canary result for {domain}: {canary_result}")
-
-        # abort if there are a suspiciously high number of results
-        # (the results are over 2000, and this is more than 20 percent of the input size)
-        if len(results) > 2000:
-            if len(results) / len(subdomains) > 0.2:
-                self.info(
-                    f"Aborting because the number of results ({len(results):,}) is suspiciously high for the length of the wordlist ({len(subdomains):,})"
-                )
-                return []
-            else:
-                self.info(
-                    f"{len(results):,} results returned from massdns against {domain} (wordlist size = {len(subdomains):,})"
-                )
-
-        # everything checks out
-        return results
-
-    async def _canary_check(self, domain, num_checks=50):
-        random_subdomains = list(self.gen_random_subdomains(num_checks))
-        self.verbose(f"Testing {len(random_subdomains):,} canaries against {domain}")
-        canary_results = [h async for h, d, r in self._massdns(domain, random_subdomains)]
-        self.log.trace(f"canary results for {domain}: {canary_results}")
-        resolved_canaries = self.helpers.resolve_batch(canary_results)
-        self.log.trace(f"resolved canary results for {domain}: {canary_results}")
-        async for query, result in resolved_canaries:
-            if result:
-                await resolved_canaries.aclose()
-                result = f"{query}:{result}"
-                self.log.trace(f"Found false positive: {result}")
-                return result
-        self.log.trace(f"Passed canary check for {domain}")
-        return False
-
-    async def _massdns(self, domain, subdomains):
-        """
-        {
-          "name": "www.blacklanternsecurity.com.",
-          "type": "A",
-          "class": "IN",
-          "status": "NOERROR",
-          "data": {
-            "answers": [
-              {
-                "ttl": 3600,
-                "type": "CNAME",
-                "class": "IN",
-                "name": "www.blacklanternsecurity.com.",
-                "data": "blacklanternsecurity.github.io."
-              },
-              {
-                "ttl": 3600,
-                "type": "A",
-                "class": "IN",
-                "name": "blacklanternsecurity.github.io.",
-                "data": "185.199.108.153"
-              }
-            ]
-          },
-          "resolver": "168.215.165.186:53"
-        }
-        """
-        if self.scan.stopping:
-            return
-
-        command = (
-            "massdns",
-            "-r",
-            self.resolver_file,
-            "-s",
-            self.max_resolvers,
-            "-t",
-            "A",
-            "-o",
-            "J",
-            "-q",
-        )
-        subdomains = self.gen_subdomains(subdomains, domain)
-        hosts_yielded = set()
-        async for line in self.run_process_live(command, stderr=subprocess.DEVNULL, input=subdomains):
-            try:
-                j = json.loads(line)
-            except json.decoder.JSONDecodeError:
-                self.debug(f"Failed to decode line: {line}")
-                continue
-            answers = j.get("data", {}).get("answers", [])
-            if type(answers) == list and len(answers) > 0:
-                answer = answers[0]
-                hostname = answer.get("name", "").strip(".").lower()
-                if hostname.endswith(f".{domain}"):
-                    data = answer.get("data", "")
-                    rdtype = answer.get("type", "").upper()
-                    # avoid garbage answers like this:
-                    # 8AAAA queries have been locally blocked by dnscrypt-proxy/Set block_ipv6 to false to disable this feature
-                    if data and rdtype and not " " in data:
-                        hostname_hash = hash(hostname)
-                        if hostname_hash not in hosts_yielded:
-                            hosts_yielded.add(hostname_hash)
-                            yield hostname, data, rdtype
-
-    async def finish(self):
-        found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True)
-        # if we have a lot of rounds to make, don't try mutations on less-populated domains
-        trimmed_found = []
-        if found:
-            avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50])
-            for i, (domain, subdomains) in enumerate(found):
-                # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains
-                if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)):
-                    trimmed_found.append((domain, subdomains))
-                else:
-                    self.verbose(
-                        f"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})"
-                    )
-
-        base_mutations = set()
-        found_mutations = False
-        try:
-            for i, (domain, subdomains) in enumerate(trimmed_found):
-                self.verbose(f"{domain} has {len(subdomains):,} subdomains")
-                # keep looping as long as we're finding things
-                while 1:
-                    max_mem_percent = 90
-                    mem_status = self.helpers.memory_status()
-                    # abort if we don't have the memory
-                    mem_percent = mem_status.percent
-                    if mem_percent > max_mem_percent:
-                        free_memory = mem_status.available
-                        free_memory_human = self.helpers.bytes_to_human(free_memory)
-                        assert (
-                            False
-                        ), f"Cannot proceed with DNS mutations because system memory is at {mem_percent:.1f}% ({free_memory_human} remaining)"
-
-                    query = domain
-                    domain_hash = hash(domain)
-                    if self.scan.stopping:
-                        return
-
-                    mutations = set(base_mutations)
-
-                    def add_mutation(_domain_hash, m):
-                        h = hash((_domain_hash, m))
-                        if h not in self.mutations_tried:
-                            self.mutations_tried.add(h)
-                            mutations.add(m)
-
-                    # try every subdomain everywhere else
-                    for _domain, _subdomains in found:
-                        if _domain == domain:
-                            continue
-                        for s in _subdomains:
-                            first_segment = s.split(".")[0]
-                            # skip stuff with lots of numbers (e.g. PTRs)
-                            digits = self.digit_regex.findall(first_segment)
-                            excessive_digits = len(digits) > 2
-                            long_digits = any(len(d) > 3 for d in digits)
-                            if excessive_digits or long_digits:
-                                continue
-                            add_mutation(domain_hash, first_segment)
-                            for word in self.helpers.extract_words(
-                                first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes
-                            ):
-                                add_mutation(domain_hash, word)
-
-                    # numbers + devops mutations
-                    for mutation in self.helpers.word_cloud.mutations(
-                        subdomains, cloud=False, numbers=3, number_padding=1
-                    ):
-                        for delimiter in ("", ".", "-"):
-                            m = delimiter.join(mutation).lower()
-                            add_mutation(domain_hash, m)
-
-                    # special dns mutator
-                    for subdomain in self.helpers.word_cloud.dns_mutator.mutations(
-                        subdomains, max_mutations=self.max_mutations
-                    ):
-                        add_mutation(domain_hash, subdomain)
-
-                    if mutations:
-                        self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})")
-                        results = list(await self.massdns(query, mutations))
-                        for hostname in results:
-                            source_event = self.source_events.get(hostname)
-                            if source_event is None:
-                                self.warning(f"Could not correlate source event from: {hostname}")
-                                source_event = self.scan.root_event
-                            await self.emit_result(
-                                hostname, source_event, query, tags=[f"mutation-{self._mutation_run}"]
-                            )
-                        if results:
-                            found_mutations = True
-                            continue
-                    break
-        except AssertionError as e:
-            self.warning(e)
-
-        if found_mutations:
-            self._mutation_run += 1
-
-    def add_found(self, host):
-        if not isinstance(host, str):
-            host = host.data
-        if self.helpers.is_subdomain(host):
-            subdomain, domain = host.split(".", 1)
-            is_ptr = self.helpers.is_ptr(subdomain)
-            in_scope = self.scan.in_scope(domain)
-            if in_scope and not is_ptr:
-                try:
-                    self.found[domain].add(subdomain)
-                except KeyError:
-                    self.found[domain] = set((subdomain,))
-
-    async def gen_subdomains(self, prefixes, domain):
-        for p in prefixes:
-            d = f"{p}.{domain}"
-            yield d
-
-    def gen_random_subdomains(self, n=50):
-        delimiters = (".", "-")
-        lengths = list(range(3, 8))
-        for i in range(0, max(0, n - 5)):
-            d = delimiters[i % len(delimiters)]
-            l = lengths[i % len(lengths)]
-            segments = list(random.choice(self.devops_mutations) for _ in range(l))
-            segments.append(self.helpers.rand_string(length=8, digits=False))
-            subdomain = d.join(segments)
-            yield subdomain
-        for _ in range(5):
-            yield self.helpers.rand_string(length=8, digits=False)
-
-    def has_excessive_digits(self, d):
-        """
-        Identifies dns names with excessive numbers, e.g.:
-            - w1-2-3.evilcorp.com
-            - ptr1234.evilcorp.com
-        """
-        digits = self.digit_regex.findall(d)
-        excessive_digits = len(digits) > 2
-        long_digits = any(len(d) > 3 for d in digits)
-        if excessive_digits or long_digits:
-            return True
-        return False
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 790b35515..3c65dfa34 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -16,16 +16,39 @@ class subdomain_enum(BaseModule):
 
     # set module error state after this many failed requests in a row
     abort_after_failures = 5
+
     # whether to reject wildcard DNS_NAMEs
     reject_wildcards = "strict"
-    # this helps combat rate limiting by ensuring that a query doesn't execute
+
+    # set qsize to 1. this helps combat rate limiting by ensuring that a query doesn't execute
     # until the queue is ready to receive its results
     _qsize = 1
 
+    # how to deduplicate incoming events
+    # options:
+    #   "root_domain": if a dns name has already been tried, don't try any of its children
+    #   "parent_domain": always try a domain unless its direct parent has already been tried
+    dedup_strategy = "root_domain"
+
     async def setup(self):
-        self.processed = set()
+        strict_scope = self.dedup_strategy == "parent_domain"
+        self.processed = self.helpers.make_target(strict_scope=strict_scope)
         return True
 
+    async def filter_event(self, event):
+        """
+        This filter_event is used across many modules
+        """
+        query = self.make_query(event)
+        # reject if already processed
+        if query in self.processed:
+            return False, "Event was already processed"
+        eligible, reason = await self.eligible_for_enumeration(event)
+        if eligible:
+            self.processed.add(query)
+            return True, reason
+        return False, reason
+
     async def handle_event(self, event):
         query = self.make_query(event)
         results = await self.query(query)
@@ -91,20 +114,6 @@ async def _is_wildcard(self, query):
                     return True
         return False
 
-    async def filter_event(self, event):
-        """
-        This filter_event is used across many modules
-        """
-        query = self.make_query(event)
-        # reject if already processed
-        if self.already_processed(query):
-            return False, "Event was already processed"
-        eligible, reason = await self.eligible_for_enumeration(event)
-        if eligible:
-            self.processed.add(hash(query))
-            return True, reason
-        return False, reason
-
     async def eligible_for_enumeration(self, event):
         query = self.make_query(event)
         # check if wildcard
@@ -128,12 +137,6 @@ async def eligible_for_enumeration(self, event):
                     return False, "Event is both a cloud resource and a wildcard domain"
         return True, ""
 
-    def already_processed(self, hostname):
-        for parent in self.helpers.domain_parents(hostname, include_self=True):
-            if hash(parent) in self.processed:
-                return True
-        return False
-
     async def abort_if(self, event):
         # this helps weed out unwanted results when scanning IP_RANGES and wildcard domains
         if "in-scope" not in event.tags:
diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 57eb11a1a..9055d03ba 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -303,9 +303,9 @@ def merge(self, other):
         self.explicit_output_modules.update(other.explicit_output_modules)
         self.flags.update(other.flags)
         # scope
-        self.target.add_target(other.target)
-        self.whitelist.add_target(other.whitelist)
-        self.blacklist.add_target(other.blacklist)
+        self.target.add(other.target)
+        self.whitelist.add(other.whitelist)
+        self.blacklist.add(other.blacklist)
         self.strict_scope = self.strict_scope or other.strict_scope
         for t in (self.target, self.whitelist):
             t.strict_scope = self.strict_scope
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index b19d1b6a6..878e80846 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -93,11 +93,11 @@ def __init__(self, *targets, strict_scope=False):
         if len(targets) > 0:
             log.verbose(f"Creating events from {len(targets):,} targets")
         for t in targets:
-            self.add_target(t)
+            self.add(t)
 
         self._hash = None
 
-    def add_target(self, t, event_type=None):
+    def add(self, t, event_type=None):
         """
         Add a target or merge events from another Target object into this Target.
 
@@ -108,7 +108,7 @@ def add_target(self, t, event_type=None):
             _events (dict): The dictionary is updated to include the new target's events.
 
         Examples:
-            >>> target.add_target('example.com')
+            >>> target.add('example.com')
 
         Notes:
             - If `t` is of the same class as this Target, all its events are merged.
diff --git a/bbot/test/test_step_2/module_tests/test_module_dastardly.py b/bbot/test/test_step_2/module_tests/test_module_dastardly.py
index ed4c20e5c..fe9de5d6c 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dastardly.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dastardly.py
@@ -44,7 +44,7 @@ async def setup_after_prep(self, module_test):
 
         # get docker IP
         docker_ip = await self.get_docker_ip(module_test)
-        module_test.scan.target.add_target(docker_ip)
+        module_test.scan.target.add(docker_ip)
 
         # replace 127.0.0.1 with docker host IP to allow dastardly access to local http server
         old_filter_event = module_test.module.filter_event
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
index 5850fbd49..3d3d670e1 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
@@ -2,22 +2,47 @@
 
 
 class TestDNSCommonSRV(ModuleTestBase):
-    targets = ["blacklanternsecurity.notreal"]
     config_overrides = {"dns_resolution": True}
 
     async def setup_after_prep(self, module_test):
+
+        old_run_live = module_test.scan.helpers.run_live
+
+        async def new_run_live(*command, check=False, text=True, **kwargs):
+            if "massdns" in command[:2]:
+                yield """{"name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","type":"SRV","class":"IN","status":"NOERROR","rx_ts":1713974911725326170,"data":{"answers":[{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."},{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."}]},"flags":["rd","ra"],"resolver":"195.226.187.130:53","proto":"UDP"}"""
+            else:
+                async for _ in old_run_live(*command, check=False, text=True, **kwargs):
+                    yield _
+
+        module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", new_run_live)
+
         await module_test.mock_dns(
             {
-                "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal": {
-                    "SRV": ["0 100 3268 asdf.blacklanternsecurity.notreal"]
-                },
-                "asdf.blacklanternsecurity.notreal": {"A": "1.2.3.4"},
+                "blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "_ldap._tcp.gc._msdcs.blacklanternsecurity.com": {"SRV": ["0 100 3268 asdf.blacklanternsecurity.com"]},
+                "asdf.blacklanternsecurity.com": {"A": ["1.2.3.5"]},
             }
         )
 
     def check(self, module_test, events):
-        assert any(
-            e.data == "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal" for e in events
-        ), "Failed to detect subdomain"
-        assert any(e.data == "asdf.blacklanternsecurity.notreal" for e in events), "Failed to detect subdomain"
-        assert not any(e.data == "_ldap._tcp.dc._msdcs.blacklanternsecurity.notreal" for e in events), "False positive"
+        assert len(events) == 4
+        assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "_ldap._tcp.gc._msdcs.blacklanternsecurity.com"
+                and str(e.module) == "dnscommonsrv"
+            ]
+        ), "Failed to detect subdomain 1"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "asdf.blacklanternsecurity.com"
+                and str(e.module) != "dnscommonsrv"
+            ]
+        ), "Failed to detect subdomain 2"
diff --git a/bbot/test/test_step_2/module_tests/test_module_massdns.py b/bbot/test/test_step_2/module_tests/test_module_massdns.py
deleted file mode 100644
index 1b4543788..000000000
--- a/bbot/test/test_step_2/module_tests/test_module_massdns.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from .base import ModuleTestBase, tempwordlist
-
-
-class TestMassdns(ModuleTestBase):
-    subdomain_wordlist = tempwordlist(["www", "asdf"])
-    config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist)}}}
-
-    def check(self, module_test, events):
-        assert any(e.data == "www.blacklanternsecurity.com" for e in events)
-        assert not any(e.data == "asdf.blacklanternsecurity.com" for e in events)

From f5ad756f269eae60c8f0b413d9abafbbda3561a2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 24 Apr 2024 13:42:49 -0400
Subject: [PATCH 009/220] wip dnsbrute rework

---
 bbot/core/helpers/dns/brute.py                | 180 ++++++++++++++++++
 bbot/modules/dnsbrute.py                      |  50 +++++
 .../module_tests/test_module_dnsbrute.py      |  78 ++++++++
 3 files changed, 308 insertions(+)
 create mode 100644 bbot/core/helpers/dns/brute.py
 create mode 100644 bbot/modules/dnsbrute.py
 create mode 100644 bbot/test/test_step_2/module_tests/test_module_dnsbrute.py

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
new file mode 100644
index 000000000..5bfa7b2e0
--- /dev/null
+++ b/bbot/core/helpers/dns/brute.py
@@ -0,0 +1,180 @@
+import json
+import random
+import asyncio
+import logging
+import subprocess
+
+
+class DNSBrute:
+    """
+    Helper for DNS brute-forcing.
+
+    Examples:
+    >>> domain = "evilcorp.com"
+    >>> subdomains = ["www", "mail"]
+    >>> results = await self.helpers.dns.brute(self, domain, subdomains)
+    """
+
+    nameservers_url = (
+        "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
+    )
+
+    def __init__(self, parent_helper):
+        self.parent_helper = parent_helper
+        self.log = logging.getLogger("bbot.helper.dns.brute")
+        self.num_canaries = 100
+        self.max_resolvers = 1000
+        self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
+        self.digit_regex = self.parent_helper.re.compile(r"\d+")
+        self._resolver_file = None
+        self._dnsbrute_lock = asyncio.Lock()
+
+    async def __call__(self, *args, **kwargs):
+        return await self.dnsbrute(*args, **kwargs)
+
+    async def dnsbrute(self, module, domain, subdomains, type=None):
+        subdomains = list(subdomains)
+
+        if type is None:
+            type = "A"
+        type = str(type).strip().upper()
+
+        domain_wildcard_rdtypes = set()
+        for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items():
+            for rdtype, results in rdtypes.items():
+                if results:
+                    domain_wildcard_rdtypes.add(rdtype)
+        if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]):
+            self.log.info(
+                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
+            )
+            return []
+        else:
+            self.log.trace(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
+
+        canaries = self.gen_random_subdomains(self.num_canaries)
+        canaries_list = list(canaries)
+        canaries_pre = canaries_list[: int(self.num_canaries / 2)]
+        canaries_post = canaries_list[int(self.num_canaries / 2) :]
+        # sandwich subdomains between canaries
+        subdomains = canaries_pre + subdomains + canaries_post
+
+        results = []
+        canaries_triggered = []
+        async for hostname, ip, rdtype in self._massdns(module, domain, subdomains, rdtype=type):
+            sub = hostname.split(domain)[0]
+            if sub in canaries:
+                canaries_triggered.append(sub)
+            else:
+                results.append(hostname)
+
+        if canaries_triggered > 5:
+            self.log.info(
+                f"Aborting massdns on {domain} due to false positive: ({len(canaries_triggered):,} canaries triggered - {','.join(canaries_triggered)})"
+            )
+            return []
+
+        # everything checks out
+        return results
+
+    async def _massdns(self, module, domain, subdomains, rdtype):
+        """
+        {
+            "name": "www.blacklanternsecurity.com.",
+            "type": "A",
+            "class": "IN",
+            "status": "NOERROR",
+            "data": {
+            "answers": [
+                {
+                "ttl": 3600,
+                "type": "CNAME",
+                "class": "IN",
+                "name": "www.blacklanternsecurity.com.",
+                "data": "blacklanternsecurity.github.io."
+                },
+                {
+                "ttl": 3600,
+                "type": "A",
+                "class": "IN",
+                "name": "blacklanternsecurity.github.io.",
+                "data": "185.199.108.153"
+                }
+            ]
+            },
+            "resolver": "168.215.165.186:53"
+        }
+        """
+        resolver_file = await self.resolver_file()
+        command = (
+            "massdns",
+            "-r",
+            resolver_file,
+            "-s",
+            self.max_resolvers,
+            "-t",
+            rdtype,
+            "-o",
+            "J",
+            "-q",
+        )
+        subdomains = self.gen_subdomains(subdomains, domain)
+        hosts_yielded = set()
+        async with self._dnsbrute_lock:
+            async for line in module.run_process_live(*command, stderr=subprocess.DEVNULL, input=subdomains):
+                self.log.critical(line)
+                try:
+                    j = json.loads(line)
+                except json.decoder.JSONDecodeError:
+                    self.log.debug(f"Failed to decode line: {line}")
+                    continue
+                answers = j.get("data", {}).get("answers", [])
+                if type(answers) == list and len(answers) > 0:
+                    answer = answers[0]
+                    hostname = answer.get("name", "").strip(".").lower()
+                    if hostname.endswith(f".{domain}"):
+                        data = answer.get("data", "")
+                        rdtype = answer.get("type", "").upper()
+                        if data and rdtype:
+                            hostname_hash = hash(hostname)
+                            if hostname_hash not in hosts_yielded:
+                                hosts_yielded.add(hostname_hash)
+                                yield hostname, data, rdtype
+
+    async def gen_subdomains(self, prefixes, domain):
+        for p in prefixes:
+            d = f"{p}.{domain}"
+            yield d
+
+    async def resolver_file(self):
+        if self._resolver_file is None:
+            self._resolver_file = await self.parent_helper.wordlist(
+                self.nameservers_url,
+                cache_hrs=24 * 7,
+            )
+        return self._resolver_file
+
+    def gen_random_subdomains(self, n=50):
+        delimiters = (".", "-")
+        lengths = list(range(3, 8))
+        for i in range(0, max(0, n - 5)):
+            d = delimiters[i % len(delimiters)]
+            l = lengths[i % len(lengths)]
+            segments = list(random.choice(self.devops_mutations) for _ in range(l))
+            segments.append(self.parent_helper.rand_string(length=8, digits=False))
+            subdomain = d.join(segments)
+            yield subdomain
+        for _ in range(5):
+            yield self.parent_helper.rand_string(length=8, digits=False)
+
+    def has_excessive_digits(self, d):
+        """
+        Identifies dns names with excessive numbers, e.g.:
+            - w1-2-3.evilcorp.com
+            - ptr1234.evilcorp.com
+        """
+        is_ptr = self.parent_helper.is_ptr(d)
+        digits = self.digit_regex.findall(d)
+        excessive_digits = len(digits) > 2
+        long_digits = any(len(d) > 3 for d in digits)
+        return is_ptr or excessive_digits or long_digits
diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py
new file mode 100644
index 000000000..2df3a48d6
--- /dev/null
+++ b/bbot/modules/dnsbrute.py
@@ -0,0 +1,50 @@
+from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+
+class dnsbrute(subdomain_enum):
+    flags = ["subdomain-enum", "passive", "aggressive"]
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    meta = {"description": "Brute-force subdomains with massdns + static wordlist"}
+    options = {
+        "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt",
+        "max_depth": 5,
+    }
+    options_desc = {
+        "wordlist": "Subdomain wordlist URL",
+        "max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com",
+    }
+    deps_common = ["massdns"]
+    reject_wildcards = "strict"
+    dedup_strategy = "parent_domain"
+    _qsize = 10000
+
+    async def setup(self):
+        self.max_depth = max(1, self.config.get("max_depth", 5))
+        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
+        self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
+        return await super().setup()
+
+    async def eligible_for_enumeration(self, event):
+        eligible, reason = await super().eligible_for_enumeration(event)
+        query = self.make_query(event)
+
+        # limit brute force depth
+        subdomain_depth = self.helpers.subdomain_depth(query) + 1
+        if subdomain_depth > self.max_depth:
+            eligible = False
+            reason = f"subdomain depth of *.{query} ({subdomain_depth}) > max_depth ({self.max_depth})"
+
+        # don't brute-force things that look like autogenerated PTRs
+        if self.helpers.dns.brute.has_excessive_digits(query):
+            eligible = False
+            reason = f'"{query}" looks like an autogenerated PTR'
+
+        return eligible, reason
+
+    async def handle_event(self, event):
+        self.hugewarning(event)
+        query = self.make_query(event)
+        self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
+        for hostname in await self.helpers.dns.brute(self, query, self.subdomain_list):
+            await self.emit_event(hostname, "DNS_NAME", source=event)
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
new file mode 100644
index 000000000..664539bb4
--- /dev/null
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
@@ -0,0 +1,78 @@
+from .base import ModuleTestBase, tempwordlist
+
+
+class TestDnsbrute(ModuleTestBase):
+    subdomain_wordlist = tempwordlist(["www", "asdf"])
+    config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist), "max_depth": 3}}}
+
+    async def setup_after_prep(self, module_test):
+
+        old_run_live = module_test.scan.helpers.run_live
+
+        async def new_run_live(*command, check=False, text=True, **kwargs):
+            if "massdns" in command[:2]:
+                yield """{"name": "www-test.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "www-test.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
+            else:
+                async for _ in old_run_live(*command, check=False, text=True, **kwargs):
+                    yield _
+
+        module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", new_run_live)
+
+        await module_test.mock_dns(
+            {
+                "www-test.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+            }
+        )
+
+        # test recursive brute-force event filtering
+        event = module_test.scan.make_event("evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == True
+        assert "evilcorp.com" in module_test.module.processed
+        assert not "com" in module_test.module.processed
+        event = module_test.scan.make_event("evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == False
+        assert reason == "Event was already processed"
+        event = module_test.scan.make_event("www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == False
+        assert reason == "Event was already processed"
+        event = module_test.scan.make_event("test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == True
+        event = module_test.scan.make_event("test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == False
+        assert reason == "Event was already processed"
+        event = module_test.scan.make_event(
+            "asdf.test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event
+        )
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == True
+        event = module_test.scan.make_event(
+            "wat.asdf.test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event
+        )
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == False
+        assert reason == f"subdomain depth of *.asdf.test.www.evilcorp.com (4) > max_depth (3)"
+        event = module_test.scan.make_event(
+            "hmmm.ptr1234.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event
+        )
+        event.scope_distance = 0
+        result, reason = await module_test.module.filter_event(event)
+        assert result == False
+        assert reason == f'"ptr1234.evilcorp.com" looks like an autogenerated PTR'
+
+    def check(self, module_test, events):
+        assert len(events) == 3
+        assert 1 == len(
+            [e for e in events if e.data == "www-test.blacklanternsecurity.com" and str(e.module) == "massdns"]
+        )

From 8bfb557505b030178d50279e7e3a9b9153fe77c5 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 24 Apr 2024 14:19:04 -0400
Subject: [PATCH 010/220] rename massdns --> dnsbrute

---
 bbot/test/test_step_1/test_cli.py             | 32 +++++++++----------
 bbot/test/test_step_1/test_modules_basic.py   | 14 ++++----
 bbot/test/test_step_1/test_presets.py         | 30 ++++++++---------
 .../module_tests/test_module_dnsbrute.py      |  4 +--
 docs/comparison.md                            |  2 +-
 docs/scanning/presets.md                      |  3 --
 6 files changed, 41 insertions(+), 44 deletions(-)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 0a34b4faa..32e761679 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -133,7 +133,7 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     assert "| bool" in caplog.text
     assert "| emit URLs in addition to DNS_NAMEs" in caplog.text
     assert "| False" in caplog.text
-    assert "| modules.massdns.wordlist" in caplog.text
+    assert "| modules.dnsbrute.wordlist" in caplog.text
     assert "| modules.robots.include_allow" in caplog.text
 
     # list module options by flag
@@ -146,17 +146,17 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     assert "| bool" in caplog.text
     assert "| emit URLs in addition to DNS_NAMEs" in caplog.text
     assert "| False" in caplog.text
-    assert "| modules.massdns.wordlist" in caplog.text
+    assert "| modules.dnsbrute.wordlist" in caplog.text
     assert not "| modules.robots.include_allow" in caplog.text
 
     # list module options by module
     caplog.clear()
     assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdns", "-lmo"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-m", "dnsbrute", "-lmo"])
     result = await cli._main()
     assert result == None
     assert not "| modules.wayback.urls" in caplog.text
-    assert "| modules.massdns.wordlist" in caplog.text
+    assert "| modules.dnsbrute.wordlist" in caplog.text
     assert not "| modules.robots.include_allow" in caplog.text
 
     # list flags
@@ -219,7 +219,7 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     monkeypatch.setattr("sys.argv", ["bbot", "-l"])
     result = await cli._main()
     assert result == None
-    assert "| massdns" in caplog.text
+    assert "| dnsbrute" in caplog.text
     assert "| httpx" in caplog.text
     assert "| robots" in caplog.text
 
@@ -229,7 +229,7 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-l"])
     result = await cli._main()
     assert result == None
-    assert "| massdns" in caplog.text
+    assert "| dnsbrute" in caplog.text
     assert "| httpx" in caplog.text
     assert not "| robots" in caplog.text
 
@@ -238,7 +238,7 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-rf", "passive", "-l"])
     result = await cli._main()
     assert result == None
-    assert "| massdns" in caplog.text
+    assert "| dnsbrute" in caplog.text
     assert not "| httpx" in caplog.text
 
     # list modules by flag + excluded flag
@@ -247,16 +247,16 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-ef", "active", "-l"])
     result = await cli._main()
     assert result == None
-    assert "| massdns" in caplog.text
+    assert "| dnsbrute" in caplog.text
     assert not "| httpx" in caplog.text
 
     # list modules by flag + excluded module
     caplog.clear()
     assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "massdns", "-l"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "dnsbrute", "-l"])
     result = await cli._main()
     assert result == None
-    assert not "| massdns" in caplog.text
+    assert not "| dnsbrute" in caplog.text
     assert "| httpx" in caplog.text
 
     # unconsoleable output module
@@ -343,18 +343,18 @@ def test_cli_module_validation(monkeypatch, caplog):
     # incorrect module
     caplog.clear()
     assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-m", "massdnss"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-m", "dnsbrutes"])
     cli.main()
-    assert 'Could not find scan module "massdnss"' in caplog.text
-    assert 'Did you mean "massdns"?' in caplog.text
+    assert 'Could not find scan module "dnsbrutes"' in caplog.text
+    assert 'Did you mean "dnsbrute"?' in caplog.text
 
     # incorrect excluded module
     caplog.clear()
     assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-em", "massdnss"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-em", "dnsbrutes"])
     cli.main()
-    assert 'Could not find module "massdnss"' in caplog.text
-    assert 'Did you mean "massdns"?' in caplog.text
+    assert 'Could not find module "dnsbrutes"' in caplog.text
+    assert 'Did you mean "dnsbrute"?' in caplog.text
 
     # incorrect output module
     caplog.clear()
diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 03273c0a7..08fd16eec 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -100,15 +100,15 @@ async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock):
 
     # module preloading
     all_preloaded = DEFAULT_PRESET.module_loader.preloaded()
-    assert "massdns" in all_preloaded
-    assert "DNS_NAME" in all_preloaded["massdns"]["watched_events"]
-    assert "DNS_NAME" in all_preloaded["massdns"]["produced_events"]
-    assert "subdomain-enum" in all_preloaded["massdns"]["flags"]
-    assert "wordlist" in all_preloaded["massdns"]["config"]
-    assert type(all_preloaded["massdns"]["config"]["max_resolvers"]) == int
+    assert "dnsbrute" in all_preloaded
+    assert "DNS_NAME" in all_preloaded["dnsbrute"]["watched_events"]
+    assert "DNS_NAME" in all_preloaded["dnsbrute"]["produced_events"]
+    assert "subdomain-enum" in all_preloaded["dnsbrute"]["flags"]
+    assert "wordlist" in all_preloaded["dnsbrute"]["config"]
+    assert type(all_preloaded["dnsbrute"]["config"]["max_depth"]) == int
     assert all_preloaded["sslcert"]["deps"]["pip"]
     assert all_preloaded["sslcert"]["deps"]["apt"]
-    assert all_preloaded["massdns"]["deps"]["common"]
+    assert all_preloaded["dnsbrute"]["deps"]["common"]
     assert all_preloaded["gowitness"]["deps"]["ansible"]
 
     all_flags = set()
diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py
index d84244e4f..6f0b9773f 100644
--- a/bbot/test/test_step_1/test_presets.py
+++ b/bbot/test/test_step_1/test_presets.py
@@ -584,13 +584,13 @@ def get_module_flags(p):
     preset = Preset(flags=["subdomain-enum"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    massdns_flags = preset.preloaded_module("massdns").get("flags", [])
-    assert "subdomain-enum" in massdns_flags
-    assert "passive" in massdns_flags
-    assert not "active" in massdns_flags
-    assert "aggressive" in massdns_flags
-    assert not "safe" in massdns_flags
-    assert "massdns" in [x[0] for x in module_flags]
+    dnsbrute_flags = preset.preloaded_module("dnsbrute").get("flags", [])
+    assert "subdomain-enum" in dnsbrute_flags
+    assert "passive" in dnsbrute_flags
+    assert not "active" in dnsbrute_flags
+    assert "aggressive" in dnsbrute_flags
+    assert not "safe" in dnsbrute_flags
+    assert "dnsbrute" in [x[0] for x in module_flags]
     assert "certspotter" in [x[0] for x in module_flags]
     assert "c99" in [x[0] for x in module_flags]
     assert any("passive" in flags for module, flags in module_flags)
@@ -602,7 +602,7 @@ def get_module_flags(p):
     preset = Preset(flags=["subdomain-enum"], require_flags=["passive"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert "massdns" in [x[0] for x in module_flags]
+    assert "dnsbrute" in [x[0] for x in module_flags]
     assert all("passive" in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
     assert any("safe" in flags for module, flags in module_flags)
@@ -612,17 +612,17 @@ def get_module_flags(p):
     preset = Preset(flags=["subdomain-enum"], exclude_flags=["active"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert "massdns" in [x[0] for x in module_flags]
+    assert "dnsbrute" in [x[0] for x in module_flags]
     assert all("passive" in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
     assert any("safe" in flags for module, flags in module_flags)
     assert any("aggressive" in flags for module, flags in module_flags)
 
     # enable by flag, one excluded module
-    preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns"]).bake()
+    preset = Preset(flags=["subdomain-enum"], exclude_modules=["dnsbrute"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "massdns" in [x[0] for x in module_flags]
+    assert not "dnsbrute" in [x[0] for x in module_flags]
     assert any("passive" in flags for module, flags in module_flags)
     assert any("active" in flags for module, flags in module_flags)
     assert any("safe" in flags for module, flags in module_flags)
@@ -632,7 +632,7 @@ def get_module_flags(p):
     preset = Preset(flags=["subdomain-enum"], require_flags=["safe", "passive"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "massdns" in [x[0] for x in module_flags]
+    assert not "dnsbrute" in [x[0] for x in module_flags]
     assert all("passive" in flags and "safe" in flags for module, flags in module_flags)
     assert all("active" not in flags and "aggressive" not in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
@@ -642,17 +642,17 @@ def get_module_flags(p):
     preset = Preset(flags=["subdomain-enum"], exclude_flags=["aggressive", "active"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "massdns" in [x[0] for x in module_flags]
+    assert not "dnsbrute" in [x[0] for x in module_flags]
     assert all("passive" in flags and "safe" in flags for module, flags in module_flags)
     assert all("active" not in flags and "aggressive" not in flags for module, flags in module_flags)
     assert not any("active" in flags for module, flags in module_flags)
     assert not any("aggressive" in flags for module, flags in module_flags)
 
     # enable by flag, multiple excluded modules
-    preset = Preset(flags=["subdomain-enum"], exclude_modules=["massdns", "c99"]).bake()
+    preset = Preset(flags=["subdomain-enum"], exclude_modules=["dnsbrute", "c99"]).bake()
     assert len(preset.modules) > 25
     module_flags = list(get_module_flags(preset))
-    assert not "massdns" in [x[0] for x in module_flags]
+    assert not "dnsbrute" in [x[0] for x in module_flags]
     assert "certspotter" in [x[0] for x in module_flags]
     assert not "c99" in [x[0] for x in module_flags]
     assert any("passive" in flags for module, flags in module_flags)
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
index 664539bb4..2d301da94 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
@@ -3,7 +3,7 @@
 
 class TestDnsbrute(ModuleTestBase):
     subdomain_wordlist = tempwordlist(["www", "asdf"])
-    config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist), "max_depth": 3}}}
+    config_overrides = {"modules": {"dnsbrute": {"wordlist": str(subdomain_wordlist), "max_depth": 3}}}
 
     async def setup_after_prep(self, module_test):
 
@@ -74,5 +74,5 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
     def check(self, module_test, events):
         assert len(events) == 3
         assert 1 == len(
-            [e for e in events if e.data == "www-test.blacklanternsecurity.com" and str(e.module) == "massdns"]
+            [e for e in events if e.data == "www-test.blacklanternsecurity.com" and str(e.module) == "dnsbrute"]
         )
diff --git a/docs/comparison.md b/docs/comparison.md
index 3226036f1..183e84319 100644
--- a/docs/comparison.md
+++ b/docs/comparison.md
@@ -2,7 +2,7 @@
 
 BBOT does a lot more than just subdomain enumeration. However, subdomain enumeration is arguably the most important part of OSINT, and since there's so many subdomain enumeration tools out there, they're the easiest class of tool to compare it to.
 
-Thanks to BBOT's recursive nature (and its `massdns` module with its NLP-powered subdomain mutations), it typically finds about 20-25% more than other tools such as `Amass` or `theHarvester`. This holds true even for larger targets like `delta.com` (1000+ subdomains):
+Thanks to BBOT's recursive nature (and its `dnsbrute_mutations` module with its NLP-powered subdomain mutations), it typically finds about 20-25% more than other tools such as `Amass` or `theHarvester`. This holds true especially for larger targets like `delta.com` (1000+ subdomains):
 
 ### Subdomains Found
 
diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md
index f19e27550..3d8f47a9b 100644
--- a/docs/scanning/presets.md
+++ b/docs/scanning/presets.md
@@ -86,9 +86,6 @@ config:
       api_key: 21a270d5f59c9b05813a72bb41707266
     virustotal:
       api_key: 4f41243847da693a4f356c0486114bc6
-    # other module config options
-    massdns:
-      max_resolvers: 5000
 ```
 
 To execute your custom preset, you do:

From 7e6a8edc4a889a2307b596f641776b7924e7be6b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 24 Apr 2024 14:20:32 -0400
Subject: [PATCH 011/220] fix tests

---
 bbot/core/helpers/dns/brute.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
index 5bfa7b2e0..da0ca0931 100644
--- a/bbot/core/helpers/dns/brute.py
+++ b/bbot/core/helpers/dns/brute.py
@@ -68,7 +68,7 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
             else:
                 results.append(hostname)
 
-        if canaries_triggered > 5:
+        if len(canaries_triggered) > 5:
             self.log.info(
                 f"Aborting massdns on {domain} due to false positive: ({len(canaries_triggered):,} canaries triggered - {','.join(canaries_triggered)})"
             )

From 2aea9b109464111dad7b89014d88a6d3ca2cd733 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 24 Apr 2024 16:54:13 -0400
Subject: [PATCH 012/220] add bloom filter

---
 bbot/core/helpers/bloom.py                 | 45 ++++++++++++++
 bbot/core/helpers/helper.py                |  5 ++
 bbot/test/test_step_1/test_bloom_filter.py | 71 ++++++++++++++++++++++
 3 files changed, 121 insertions(+)
 create mode 100644 bbot/core/helpers/bloom.py
 create mode 100644 bbot/test/test_step_1/test_bloom_filter.py

diff --git a/bbot/core/helpers/bloom.py b/bbot/core/helpers/bloom.py
new file mode 100644
index 000000000..1f454ce0c
--- /dev/null
+++ b/bbot/core/helpers/bloom.py
@@ -0,0 +1,45 @@
+import mmh3
+from bitarray import bitarray
+
+
+class BloomFilter:
+    """
+    Simple bloom filter implementation capable of rougly 200K lookups/s.
+
+    BBOT uses bloom filters in scenarios like dns brute-forcing, where it's useful to keep track
+    of which mutations have been tried so far.
+
+    A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.
+    A python hash is 36 bytes. So if you wanted to store these in a set, this would take up
+    36 * 10M * 2 (key+value) == 720 megabytes. So we save rougly 7 times the space.
+    """
+
+    def __init__(self, size=2**16):
+        self.size = size
+        self.bit_array = bitarray(size)
+        self.bit_array.setall(0)  # Initialize all bits to 0
+
+    def _hashes(self, item):
+        item_str = str(item).encode("utf-8")
+        return [
+            abs(hash(item)) % self.size,
+            abs(mmh3.hash(item_str)) % self.size,
+            abs(self._fnv1a_hash(item_str)) % self.size,
+        ]
+
+    def _fnv1a_hash(self, data):
+        hash = 0x811C9DC5  # 2166136261
+        for byte in data:
+            hash ^= byte
+            hash = (hash * 0x01000193) % 2**32  # 16777619
+        return hash
+
+    def add(self, item):
+        for hash_value in self._hashes(item):
+            self.bit_array[hash_value] = 1
+
+    def check(self, item):
+        return all(self.bit_array[hash_value] for hash_value in self._hashes(item))
+
+    def __contains__(self, item):
+        return self.check(item)
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index e4dc09326..56b9c3bbd 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -91,6 +91,11 @@ def __init__(self, preset):
         self.word_cloud = WordCloud(self)
         self.dummy_modules = {}
 
+    def bloom_filter(self, size):
+        from .bloom import BloomFilter
+
+        return BloomFilter(size)
+
     def interactsh(self, *args, **kwargs):
         return Interactsh(self, *args, **kwargs)
 
diff --git a/bbot/test/test_step_1/test_bloom_filter.py b/bbot/test/test_step_1/test_bloom_filter.py
new file mode 100644
index 000000000..089f4af82
--- /dev/null
+++ b/bbot/test/test_step_1/test_bloom_filter.py
@@ -0,0 +1,71 @@
+import sys
+import time
+import string
+import random
+
+
+def test_bloom_filter():
+
+    def generate_random_strings(n, length=10):
+        """Generate a list of n random strings."""
+        return ["".join(random.choices(string.ascii_letters + string.digits, k=length)) for _ in range(n)]
+
+    from bbot.scanner import Scanner
+
+    scan = Scanner()
+
+    n_items_to_add = 100000
+    n_items_to_test = 100000
+    bloom_filter_size = 8000000
+
+    # Initialize the simple bloom filter and the set
+    bloom_filter = scan.helpers.bloom_filter(size=bloom_filter_size)
+    test_set = set()
+
+    mem_size = sys.getsizeof(bloom_filter.bit_array)
+    print(f"Size of bit array: {mem_size}")
+
+    # size should be roughly 1MB
+    assert 900000 < mem_size < 1100000
+
+    # Generate random strings to add
+    print(f"Generating {n_items_to_add:,} items to add")
+    items_to_add = set(generate_random_strings(n_items_to_add))
+
+    # Generate random strings to test
+    print(f"Generating {n_items_to_test:,} items to test")
+    items_to_test = generate_random_strings(n_items_to_test)
+
+    print("Adding items")
+    start = time.time()
+    for item in items_to_add:
+        bloom_filter.add(item)
+        test_set.add(hash(item))
+    end = time.time()
+    elapsed = end - start
+    print(f"elapsed: {elapsed:.2f} ({int(n_items_to_test/elapsed)}/s)")
+    # this shouldn't take longer than 5 seconds
+    assert elapsed < 5
+
+    # make sure we have 100% accuracy
+    start = time.time()
+    for item in items_to_add:
+        assert item in bloom_filter
+    end = time.time()
+    elapsed = end - start
+    print(f"elapsed: {elapsed:.2f} ({int(n_items_to_test/elapsed)}/s)")
+    # this shouldn't take longer than 5 seconds
+    assert elapsed < 5
+
+    print("Measuring false positives")
+    # Check for false positives
+    false_positives = 0
+    for item in items_to_test:
+        if bloom_filter.check(item) and hash(item) not in test_set:
+            false_positives += 1
+    false_positive_rate = false_positives / len(items_to_test)
+
+    print(f"False positive rate: {false_positive_rate * 100:.2f}% ({false_positives}/{len(items_to_test)})")
+
+    # ensure false positives are less than .01 percent
+    assert 0 < false_positives < 10

From 6fd52718f807cc1d2b5fc6a9053da63dbb7cd32b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 24 Apr 2024 17:16:21 -0400
Subject: [PATCH 013/220] wip dnsbrute mutations

---
 bbot/core/helpers/dns/brute.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
index da0ca0931..d8996f2f4 100644
--- a/bbot/core/helpers/dns/brute.py
+++ b/bbot/core/helpers/dns/brute.py
@@ -122,7 +122,6 @@ async def _massdns(self, module, domain, subdomains, rdtype):
         hosts_yielded = set()
         async with self._dnsbrute_lock:
             async for line in module.run_process_live(*command, stderr=subprocess.DEVNULL, input=subdomains):
-                self.log.critical(line)
                 try:
                     j = json.loads(line)
                 except json.decoder.JSONDecodeError:

From 1e4927184f11d4fc57fd6181851f22966d5b00ca Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 25 Apr 2024 10:45:04 -0400
Subject: [PATCH 014/220] updates to bloom filter

---
 bbot/core/helpers/bloom.py                 | 64 ++++++++++-----
 bbot/modules/rapiddns.py                   |  2 +-
 bbot/test/test_step_1/test_bloom_filter.py | 16 ++--
 poetry.lock                                | 94 +++++++++++++++++++++-
 pyproject.toml                             |  1 +
 5 files changed, 145 insertions(+), 32 deletions(-)

diff --git a/bbot/core/helpers/bloom.py b/bbot/core/helpers/bloom.py
index 1f454ce0c..357c715c0 100644
--- a/bbot/core/helpers/bloom.py
+++ b/bbot/core/helpers/bloom.py
@@ -1,12 +1,13 @@
+import os
 import mmh3
-from bitarray import bitarray
+import mmap
 
 
 class BloomFilter:
     """
-    Simple bloom filter implementation capable of rougly 200K lookups/s.
+    Simple bloom filter implementation capable of rougly 400K lookups/s.
 
-    BBOT uses bloom filters in scenarios like dns brute-forcing, where it's useful to keep track
+    BBOT uses bloom filters in scenarios like DNS brute-forcing, where it's useful to keep track
     of which mutations have been tried so far.
 
     A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.
@@ -14,18 +15,47 @@ class BloomFilter:
     36 * 10M * 2 (key+value) == 720 megabytes. So we save rougly 7 times the space.
     """
 
-    def __init__(self, size=2**16):
-        self.size = size
-        self.bit_array = bitarray(size)
-        self.bit_array.setall(0)  # Initialize all bits to 0
+    def __init__(self, size=8000000):
+        self.size = size  # total bits
+        self.byte_size = (size + 7) // 8  # calculate byte size needed for the given number of bits
+
+        # Create an anonymous mmap region, compatible with both Windows and Unix
+        if os.name == "nt":  # Windows
+            # -1 indicates an anonymous memory map in Windows
+            self.mmap_file = mmap.mmap(-1, self.byte_size)
+        else:  # Unix/Linux
+            # Use MAP_ANONYMOUS along with MAP_SHARED
+            self.mmap_file = mmap.mmap(-1, self.byte_size, prot=mmap.PROT_WRITE, flags=mmap.MAP_ANON | mmap.MAP_SHARED)
+
+        self.clear_all_bits()
+
+    def add(self, item):
+        for hash_value in self._hashes(item):
+            index = hash_value // 8
+            position = hash_value % 8
+            current_byte = self.mmap_file[index]
+            self.mmap_file[index] = current_byte | (1 << position)
+
+    def check(self, item):
+        for hash_value in self._hashes(item):
+            index = hash_value // 8
+            position = hash_value % 8
+            current_byte = self.mmap_file[index]
+            if not (current_byte & (1 << position)):
+                return False
+        return True
+
+    def clear_all_bits(self):
+        self.mmap_file.seek(0)
+        # Write zeros across the entire mmap length
+        self.mmap_file.write(b"\x00" * self.byte_size)
 
     def _hashes(self, item):
-        item_str = str(item).encode("utf-8")
-        return [
-            abs(hash(item)) % self.size,
-            abs(mmh3.hash(item_str)) % self.size,
-            abs(self._fnv1a_hash(item_str)) % self.size,
-        ]
+        if not isinstance(item, bytes):
+            if not isinstance(item, str):
+                item = str(item)
+            item = item.encode("utf-8")
+        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
 
     def _fnv1a_hash(self, data):
         hash = 0x811C9DC5  # 2166136261
@@ -34,12 +64,8 @@ def _fnv1a_hash(self, data):
             hash = (hash * 0x01000193) % 2**32  # 16777619
         return hash
 
-    def add(self, item):
-        for hash_value in self._hashes(item):
-            self.bit_array[hash_value] = 1
-
-    def check(self, item):
-        return all(self.bit_array[hash_value] for hash_value in self._hashes(item))
+    def __del__(self):
+        self.mmap_file.close()
 
     def __contains__(self, item):
         return self.check(item)
diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py
index 088288ddb..7e634515b 100644
--- a/bbot/modules/rapiddns.py
+++ b/bbot/modules/rapiddns.py
@@ -11,7 +11,7 @@ class rapiddns(subdomain_enum):
 
     async def request_url(self, query):
         url = f"{self.base_url}/subdomain/{self.helpers.quote(query)}?full=1#result"
-        response = await self.request_with_fail_count(url)
+        response = await self.request_with_fail_count(url, timeout=self.http_timeout + 10)
         return response
 
     def parse_results(self, r, query):
diff --git a/bbot/test/test_step_1/test_bloom_filter.py b/bbot/test/test_step_1/test_bloom_filter.py
index 089f4af82..6d8e6918d 100644
--- a/bbot/test/test_step_1/test_bloom_filter.py
+++ b/bbot/test/test_step_1/test_bloom_filter.py
@@ -1,4 +1,3 @@
-import sys
 import time
 import string
 import random
@@ -20,13 +19,8 @@ def generate_random_strings(n, length=10):
 
     # Initialize the simple bloom filter and the set
     bloom_filter = scan.helpers.bloom_filter(size=bloom_filter_size)
-    test_set = set()
-
-    mem_size = sys.getsizeof(bloom_filter.bit_array)
-    print(f"Size of bit array: {mem_size}")
 
-    # size should be roughly 1MB
-    assert 900000 < mem_size < 1100000
+    test_set = set()
 
     # Generate random strings to add
     print(f"Generating {n_items_to_add:,} items to add")
@@ -63,9 +57,9 @@ def generate_random_strings(n, length=10):
     for item in items_to_test:
         if bloom_filter.check(item) and hash(item) not in test_set:
             false_positives += 1
-    false_positive_rate = false_positives / len(items_to_test)
+    false_positive_percent = false_positives / len(items_to_test) * 100
 
-    print(f"False positive rate: {false_positive_rate * 100:.2f}% ({false_positives}/{len(items_to_test)})")
+    print(f"False positive rate: {false_positive_percent:.2f}% ({false_positives}/{len(items_to_test)})")
 
-    # ensure false positives are less than .01 percent
-    assert 0 < false_positives < 10
+    # ensure false positives are less than .02 percent
+    assert false_positive_percent < 0.02
diff --git a/poetry.lock b/poetry.lock
index be6fea410..e54a2c530 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1295,6 +1295,98 @@ files = [
 griffe = ">=0.44"
 mkdocstrings = ">=0.24.2"
 
+[[package]]
+name = "mmh3"
+version = "4.1.0"
+description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions."
+optional = false
+python-versions = "*"
+files = [
+    {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be5ac76a8b0cd8095784e51e4c1c9c318c19edcd1709a06eb14979c8d850c31a"},
+    {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98a49121afdfab67cd80e912b36404139d7deceb6773a83620137aaa0da5714c"},
+    {file = "mmh3-4.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5259ac0535874366e7d1a5423ef746e0d36a9e3c14509ce6511614bdc5a7ef5b"},
+    {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5950827ca0453a2be357696da509ab39646044e3fa15cad364eb65d78797437"},
+    {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dd0f652ae99585b9dd26de458e5f08571522f0402155809fd1dc8852a613a39"},
+    {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d25548070942fab1e4a6f04d1626d67e66d0b81ed6571ecfca511f3edf07e6"},
+    {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53db8d9bad3cb66c8f35cbc894f336273f63489ce4ac416634932e3cbe79eb5b"},
+    {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75da0f615eb55295a437264cc0b736753f830b09d102aa4c2a7d719bc445ec05"},
+    {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b926b07fd678ea84b3a2afc1fa22ce50aeb627839c44382f3d0291e945621e1a"},
+    {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c5b053334f9b0af8559d6da9dc72cef0a65b325ebb3e630c680012323c950bb6"},
+    {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bf33dc43cd6de2cb86e0aa73a1cc6530f557854bbbe5d59f41ef6de2e353d7b"},
+    {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fa7eacd2b830727ba3dd65a365bed8a5c992ecd0c8348cf39a05cc77d22f4970"},
+    {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42dfd6742b9e3eec599f85270617debfa0bbb913c545bb980c8a4fa7b2d047da"},
+    {file = "mmh3-4.1.0-cp310-cp310-win32.whl", hash = "sha256:2974ad343f0d39dcc88e93ee6afa96cedc35a9883bc067febd7ff736e207fa47"},
+    {file = "mmh3-4.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:74699a8984ded645c1a24d6078351a056f5a5f1fe5838870412a68ac5e28d865"},
+    {file = "mmh3-4.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f0dc874cedc23d46fc488a987faa6ad08ffa79e44fb08e3cd4d4cf2877c00a00"},
+    {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3280a463855b0eae64b681cd5b9ddd9464b73f81151e87bb7c91a811d25619e6"},
+    {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97ac57c6c3301769e757d444fa7c973ceb002cb66534b39cbab5e38de61cd896"},
+    {file = "mmh3-4.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b6502cdb4dbd880244818ab363c8770a48cdccecf6d729ade0241b736b5ec0"},
+    {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52ba2da04671a9621580ddabf72f06f0e72c1c9c3b7b608849b58b11080d8f14"},
+    {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a5fef4c4ecc782e6e43fbeab09cff1bac82c998a1773d3a5ee6a3605cde343e"},
+    {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5135358a7e00991f73b88cdc8eda5203bf9de22120d10a834c5761dbeb07dd13"},
+    {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cff9ae76a54f7c6fe0167c9c4028c12c1f6de52d68a31d11b6790bb2ae685560"},
+    {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f02576a4d106d7830ca90278868bf0983554dd69183b7bbe09f2fcd51cf54f"},
+    {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:073d57425a23721730d3ff5485e2da489dd3c90b04e86243dd7211f889898106"},
+    {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:71e32ddec7f573a1a0feb8d2cf2af474c50ec21e7a8263026e8d3b4b629805db"},
+    {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7cbb20b29d57e76a58b40fd8b13a9130db495a12d678d651b459bf61c0714cea"},
+    {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a42ad267e131d7847076bb7e31050f6c4378cd38e8f1bf7a0edd32f30224d5c9"},
+    {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a013979fc9390abadc445ea2527426a0e7a4495c19b74589204f9b71bcaafeb"},
+    {file = "mmh3-4.1.0-cp311-cp311-win32.whl", hash = "sha256:1d3b1cdad7c71b7b88966301789a478af142bddcb3a2bee563f7a7d40519a00f"},
+    {file = "mmh3-4.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0dc6dc32eb03727467da8e17deffe004fbb65e8b5ee2b502d36250d7a3f4e2ec"},
+    {file = "mmh3-4.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9ae3a5c1b32dda121c7dc26f9597ef7b01b4c56a98319a7fe86c35b8bc459ae6"},
+    {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0033d60c7939168ef65ddc396611077a7268bde024f2c23bdc283a19123f9e9c"},
+    {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5"},
+    {file = "mmh3-4.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d82eb4defa245e02bb0b0dc4f1e7ee284f8d212633389c91f7fba99ba993f0a2"},
+    {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba245e94b8d54765e14c2d7b6214e832557e7856d5183bc522e17884cab2f45d"},
+    {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb04e2feeabaad6231e89cd43b3d01a4403579aa792c9ab6fdeef45cc58d4ec0"},
+    {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e3b1a27def545ce11e36158ba5d5390cdbc300cfe456a942cc89d649cf7e3b2"},
+    {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce0ab79ff736d7044e5e9b3bfe73958a55f79a4ae672e6213e92492ad5e734d5"},
+    {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b02268be6e0a8eeb8a924d7db85f28e47344f35c438c1e149878bb1c47b1cd3"},
+    {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:deb887f5fcdaf57cf646b1e062d56b06ef2f23421c80885fce18b37143cba828"},
+    {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99dd564e9e2b512eb117bd0cbf0f79a50c45d961c2a02402787d581cec5448d5"},
+    {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:08373082dfaa38fe97aa78753d1efd21a1969e51079056ff552e687764eafdfe"},
+    {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:54b9c6a2ea571b714e4fe28d3e4e2db37abfd03c787a58074ea21ee9a8fd1740"},
+    {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a7b1edf24c69e3513f879722b97ca85e52f9032f24a52284746877f6a7304086"},
+    {file = "mmh3-4.1.0-cp312-cp312-win32.whl", hash = "sha256:411da64b951f635e1e2284b71d81a5a83580cea24994b328f8910d40bed67276"},
+    {file = "mmh3-4.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bebc3ecb6ba18292e3d40c8712482b4477abd6981c2ebf0e60869bd90f8ac3a9"},
+    {file = "mmh3-4.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:168473dd608ade6a8d2ba069600b35199a9af837d96177d3088ca91f2b3798e3"},
+    {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:372f4b7e1dcde175507640679a2a8790185bb71f3640fc28a4690f73da986a3b"},
+    {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:438584b97f6fe13e944faf590c90fc127682b57ae969f73334040d9fa1c7ffa5"},
+    {file = "mmh3-4.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e27931b232fc676675fac8641c6ec6b596daa64d82170e8597f5a5b8bdcd3b6"},
+    {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:571a92bad859d7b0330e47cfd1850b76c39b615a8d8e7aa5853c1f971fd0c4b1"},
+    {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a69d6afe3190fa08f9e3a58e5145549f71f1f3fff27bd0800313426929c7068"},
+    {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afb127be0be946b7630220908dbea0cee0d9d3c583fa9114a07156f98566dc28"},
+    {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940d86522f36348ef1a494cbf7248ab3f4a1638b84b59e6c9e90408bd11ad729"},
+    {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dcccc4935686619a8e3d1f7b6e97e3bd89a4a796247930ee97d35ea1a39341"},
+    {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01bb9b90d61854dfc2407c5e5192bfb47222d74f29d140cb2dd2a69f2353f7cc"},
+    {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bcb1b8b951a2c0b0fb8a5426c62a22557e2ffc52539e0a7cc46eb667b5d606a9"},
+    {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6477a05d5e5ab3168e82e8b106e316210ac954134f46ec529356607900aea82a"},
+    {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:da5892287e5bea6977364b15712a2573c16d134bc5fdcdd4cf460006cf849278"},
+    {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:99180d7fd2327a6fffbaff270f760576839dc6ee66d045fa3a450f3490fda7f5"},
+    {file = "mmh3-4.1.0-cp38-cp38-win32.whl", hash = "sha256:9b0d4f3949913a9f9a8fb1bb4cc6ecd52879730aab5ff8c5a3d8f5b593594b73"},
+    {file = "mmh3-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:598c352da1d945108aee0c3c3cfdd0e9b3edef74108f53b49d481d3990402169"},
+    {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:475d6d1445dd080f18f0f766277e1237fa2914e5fe3307a3b2a3044f30892103"},
+    {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5ca07c41e6a2880991431ac717c2a049056fff497651a76e26fc22224e8b5732"},
+    {file = "mmh3-4.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ebe052fef4bbe30c0548d12ee46d09f1b69035ca5208a7075e55adfe091be44"},
+    {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaefd42e85afb70f2b855a011f7b4d8a3c7e19c3f2681fa13118e4d8627378c5"},
+    {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0ae43caae5a47afe1b63a1ae3f0986dde54b5fb2d6c29786adbfb8edc9edfb"},
+    {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6218666f74c8c013c221e7f5f8a693ac9cf68e5ac9a03f2373b32d77c48904de"},
+    {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac59294a536ba447b5037f62d8367d7d93b696f80671c2c45645fa9f1109413c"},
+    {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086844830fcd1e5c84fec7017ea1ee8491487cfc877847d96f86f68881569d2e"},
+    {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e42b38fad664f56f77f6fbca22d08450f2464baa68acdbf24841bf900eb98e87"},
+    {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d08b790a63a9a1cde3b5d7d733ed97d4eb884bfbc92f075a091652d6bfd7709a"},
+    {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:73ea4cc55e8aea28c86799ecacebca09e5f86500414870a8abaedfcbaf74d288"},
+    {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f90938ff137130e47bcec8dc1f4ceb02f10178c766e2ef58a9f657ff1f62d124"},
+    {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:aa1f13e94b8631c8cd53259250556edcf1de71738936b60febba95750d9632bd"},
+    {file = "mmh3-4.1.0-cp39-cp39-win32.whl", hash = "sha256:a3b680b471c181490cf82da2142029edb4298e1bdfcb67c76922dedef789868d"},
+    {file = "mmh3-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:fefef92e9c544a8dbc08f77a8d1b6d48006a750c4375bbcd5ff8199d761e263b"},
+    {file = "mmh3-4.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:8e2c1f6a2b41723a4f82bd5a762a777836d29d664fc0095f17910bea0adfd4a6"},
+    {file = "mmh3-4.1.0.tar.gz", hash = "sha256:a1cf25348b9acd229dda464a094d6170f47d2850a1fcb762a3b6172d2ce6ca4a"},
+]
+
+[package.extras]
+test = ["mypy (>=1.0)", "pytest (>=7.0.0)"]
+
 [[package]]
 name = "mypy-extensions"
 version = "1.0.0"
@@ -2637,4 +2729,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "ed8bb07e4ff5a5f665402db33f9016409547bef1ccb6a8c2c626c44fde075abb"
+content-hash = "baf84bb8d915bbcec435bf66a227dc0aac2dad1acc2e3f7028a19cd23f87bf1b"
diff --git a/pyproject.toml b/pyproject.toml
index 7ba00c488..4ca394188 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,6 +52,7 @@ regex = "^2024.4.16"
 unidecode = "^1.3.8"
 radixtarget = "^1.0.0.15"
 cloudcheck = "^5.0.0.350"
+mmh3 = "^4.1.0"
 
 [tool.poetry.group.dev.dependencies]
 flake8 = ">=6,<8"

From 949f9c79737decf603286fcdf3e0102f35fce219 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 25 Apr 2024 11:30:18 -0400
Subject: [PATCH 015/220] better error handling in intercept modules

---
 bbot/modules/base.py                 | 4 ++++
 bbot/test/test_step_1/test_target.py | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index c102b138d..a55ff2ae4 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -1481,6 +1481,10 @@ async def _worker(self):
             except asyncio.CancelledError:
                 self.log.trace("Worker cancelled")
                 raise
+            except BaseException as e:
+                self.critical(f"Critical failure in intercept module {self.name}: {e}")
+                self.critical(traceback.format_exc())
+                self.scan.stop()
         self.log.trace(f"Worker stopped")
 
     async def get_incoming_event(self):
diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py
index ed5c1b7ef..7d8117d52 100644
--- a/bbot/test/test_step_1/test_target.py
+++ b/bbot/test/test_step_1/test_target.py
@@ -51,12 +51,12 @@ def test_target(bbot_scanner):
     assert not "www.evilcorp.com" in strict_target
 
     target = Target()
-    target.add_target("evilcorp.com")
+    target.add("evilcorp.com")
     assert not "com" in target
     assert "evilcorp.com" in target
     assert "www.evilcorp.com" in target
     strict_target = Target(strict_scope=True)
-    strict_target.add_target("evilcorp.com")
+    strict_target.add("evilcorp.com")
     assert not "com" in strict_target
     assert "evilcorp.com" in strict_target
     assert not "www.evilcorp.com" in strict_target

From c44be0b7dad6b48779ec135480564839cea99d2f Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 25 Apr 2024 15:53:36 -0400
Subject: [PATCH 016/220] add dnsbrute_mutations module

---
 bbot/modules/dnsbrute_mutations.py            | 140 ++++++++++++++++++
 .../test_module_dnsbrute_mutations.py         |  70 +++++++++
 2 files changed, 210 insertions(+)
 create mode 100644 bbot/modules/dnsbrute_mutations.py
 create mode 100644 bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py

diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py
new file mode 100644
index 000000000..08e00d878
--- /dev/null
+++ b/bbot/modules/dnsbrute_mutations.py
@@ -0,0 +1,140 @@
+from bbot.modules.base import BaseModule
+
+
+class dnsbrute_mutations(BaseModule):
+    flags = ["subdomain-enum", "passive", "aggressive", "slow"]
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    meta = {"description": "Brute-force subdomains with massdns + target-specific mutations"}
+    options = {
+        "max_mutations": 100,
+    }
+    options_desc = {
+        "max_mutations": 100,
+    }
+    deps_common = ["massdns"]
+    _qsize = 10000
+
+    async def setup(self):
+        self.found = {}
+        self.source_events = self.helpers.make_target()
+        self.max_mutations = self.config.get("max_mutations", 500)
+        # 800M bits == 100MB bloom filter == 10M entries before false positives start emerging
+        self.mutations_tried = self.helpers.bloom_filter(800000000)
+        self._mutation_run = 1
+        return True
+
+    async def handle_event(self, event):
+        # here we don't brute-force, we just add the subdomain to our end-of-scan TODO
+        self.add_found(event)
+
+    def add_found(self, event):
+        self.source_events.add(event)
+        host = str(event.host)
+        if self.helpers.is_subdomain(host):
+            subdomain, domain = host.split(".", 1)
+            if not self.helpers.dns.brute.has_excessive_digits(subdomain):
+                try:
+                    self.found[domain].add(subdomain)
+                except KeyError:
+                    self.found[domain] = {subdomain}
+
+    async def finish(self):
+        found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True)
+        # if we have a lot of rounds to make, don't try mutations on less-populated domains
+        trimmed_found = []
+        if found:
+            avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50])
+            for i, (domain, subdomains) in enumerate(found):
+                # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains
+                if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)):
+                    trimmed_found.append((domain, subdomains))
+                else:
+                    self.verbose(
+                        f"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})"
+                    )
+
+        base_mutations = set()
+        found_mutations = False
+        try:
+            for i, (domain, subdomains) in enumerate(trimmed_found):
+                self.verbose(f"{domain} has {len(subdomains):,} subdomains")
+                # keep looping as long as we're finding things
+                while 1:
+                    query = domain
+
+                    mutations = set(base_mutations)
+
+                    def add_mutation(m):
+                        h = f"{m}.{domain}"
+                        if h not in self.mutations_tried:
+                            self.mutations_tried.add(h)
+                            mutations.add(m)
+
+                    # try every subdomain everywhere else
+                    for _domain, _subdomains in found:
+                        if _domain == domain:
+                            continue
+                        for s in _subdomains:
+                            first_segment = s.split(".")[0]
+                            # skip stuff with lots of numbers (e.g. PTRs)
+                            if self.helpers.dns.brute.has_excessive_digits(first_segment):
+                                continue
+                            add_mutation(first_segment)
+                            for word in self.helpers.extract_words(
+                                first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes
+                            ):
+                                add_mutation(word)
+
+                    # numbers + devops mutations
+                    for mutation in self.helpers.word_cloud.mutations(
+                        subdomains, cloud=False, numbers=3, number_padding=1
+                    ):
+                        for delimiter in ("", ".", "-"):
+                            m = delimiter.join(mutation).lower()
+                            add_mutation(m)
+
+                    # special dns mutator
+                    for subdomain in self.helpers.word_cloud.dns_mutator.mutations(
+                        subdomains, max_mutations=self.max_mutations
+                    ):
+                        add_mutation(subdomain)
+
+                    if mutations:
+                        self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(trimmed_found)})")
+                        results = await self.helpers.dns.brute(self, query, mutations)
+                        for hostname in results:
+                            source_event = self.source_events.get(hostname)
+                            if source_event is None:
+                                self.warning(f"Could not correlate source event from: {hostname}")
+                                self.warning(self.source_events._radix.dns_tree.root.children)
+                                self.warning(self.source_events._radix.dns_tree.root.children["com"].children)
+                                self.warning(
+                                    self.source_events._radix.dns_tree.root.children["com"].children["tesla"].children
+                                )
+                                source_event = self.scan.root_event
+                            await self.emit_event(
+                                hostname,
+                                "DNS_NAME",
+                                source=source_event,
+                                tags=[f"mutation-{self._mutation_run}"],
+                                abort_if=self.abort_if,
+                            )
+                        if results:
+                            found_mutations = True
+                            continue
+                    break
+        except AssertionError as e:
+            self.warning(e)
+
+        if found_mutations:
+            self._mutation_run += 1
+
+    def abort_if(self, event):
+        if not event.scope_distance == 0:
+            return True, "event is not in scope"
+        if "wildcard" in event.tags:
+            return True, "event is a wildcard"
+        if "unresolved" in event.tags:
+            return True, "event is unresolved"
+        return False, ""
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py
new file mode 100644
index 000000000..2a56b2b65
--- /dev/null
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py
@@ -0,0 +1,70 @@
+from .base import ModuleTestBase
+
+
+class TestDnsbrute_mutations(ModuleTestBase):
+    targets = [
+        "blacklanternsecurity.com",
+        "rrrr.blacklanternsecurity.com",
+        "asdff-ffdsa.blacklanternsecurity.com",
+        "hmmmm.test1.blacklanternsecurity.com",
+        "uuuuu.test2.blacklanternsecurity.com",
+    ]
+
+    async def setup_after_prep(self, module_test):
+
+        old_run_live = module_test.scan.helpers.run_live
+
+        async def new_run_live(*command, check=False, text=True, **kwargs):
+            if "massdns" in command[:2]:
+                _input = [l async for l in kwargs["input"]]
+                if "rrrr-test.blacklanternsecurity.com" in _input:
+                    yield """{"name": "rrrr-test.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "rrrr-test.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
+                if "rrrr-ffdsa.blacklanternsecurity.com" in _input:
+                    yield """{"name": "rrrr-ffdsa.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "rrrr-ffdsa.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
+                if "hmmmm.test2.blacklanternsecurity.com" in _input:
+                    yield """{"name": "hmmmm.test2.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "hmmmm.test2.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
+            else:
+                async for _ in old_run_live(*command, check=False, text=True, **kwargs):
+                    yield _
+
+        module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", new_run_live)
+
+        await module_test.mock_dns(
+            {
+                # targets
+                "rrrr.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "asdff-ffdsa.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "hmmmm.test1.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "uuuuu.test2.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                # devops mutation
+                "rrrr-test.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                # target-specific dns mutation
+                "rrrr-ffdsa.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                # subdomain from one subdomain on a different subdomain
+                "hmmmm.test2.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+            }
+        )
+
+    def check(self, module_test, events):
+        assert len(events) == 9
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.data == "rrrr-test.blacklanternsecurity.com" and str(e.module) == "dnsbrute_mutations"
+            ]
+        ), "Failed to find devops mutation (word_cloud)"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.data == "rrrr-ffdsa.blacklanternsecurity.com" and str(e.module) == "dnsbrute_mutations"
+            ]
+        ), "Failed to find target-specific mutation (word_cloud.dns_mutator)"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.data == "hmmmm.test2.blacklanternsecurity.com" and str(e.module) == "dnsbrute_mutations"
+            ]
+        ), "Failed to find subdomain taken from another subdomain"

From e94556b594db8551f3bd3453418df10a30bf1338 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 25 Apr 2024 16:03:59 -0400
Subject: [PATCH 017/220] fix tests

---
 bbot/modules/dnsbrute_mutations.py | 2 +-
 bbot/test/test_step_1/test_cli.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py
index 08e00d878..06fd03689 100644
--- a/bbot/modules/dnsbrute_mutations.py
+++ b/bbot/modules/dnsbrute_mutations.py
@@ -10,7 +10,7 @@ class dnsbrute_mutations(BaseModule):
         "max_mutations": 100,
     }
     options_desc = {
-        "max_mutations": 100,
+        "max_mutations": "Maximum number of target-specific mutations to try per subdomain",
     }
     deps_common = ["massdns"]
     _qsize = 10000
diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 32e761679..32e9bcadb 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -253,7 +253,7 @@ async def test_cli_args(monkeypatch, caplog, clean_default_config):
     # list modules by flag + excluded module
     caplog.clear()
     assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "dnsbrute", "-l"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "dnsbrute", "dnsbrute_mutations", "-l"])
     result = await cli._main()
     assert result == None
     assert not "| dnsbrute" in caplog.text

From a2669b0b825f81546af63955bc16a0c8e7a005de Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 25 Apr 2024 16:52:07 -0400
Subject: [PATCH 018/220] fix dnsbrute tests

---
 bbot/test/test_step_2/module_tests/test_module_dnsbrute.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
index 2d301da94..fab736cca 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
@@ -11,7 +11,7 @@ async def setup_after_prep(self, module_test):
 
         async def new_run_live(*command, check=False, text=True, **kwargs):
             if "massdns" in command[:2]:
-                yield """{"name": "www-test.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "www-test.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
+                yield """{"name": "asdf.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "asdf.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
             else:
                 async for _ in old_run_live(*command, check=False, text=True, **kwargs):
                     yield _
@@ -20,7 +20,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
 
         await module_test.mock_dns(
             {
-                "www-test.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "asdf.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
             }
         )
 
@@ -74,5 +74,5 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
     def check(self, module_test, events):
         assert len(events) == 3
         assert 1 == len(
-            [e for e in events if e.data == "www-test.blacklanternsecurity.com" and str(e.module) == "dnsbrute"]
+            [e for e in events if e.data == "asdf.blacklanternsecurity.com" and str(e.module) == "dnsbrute"]
         )

From c5de1360e8e5ccba04b23035f675a529282b7dc2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 25 Apr 2024 17:02:37 -0400
Subject: [PATCH 019/220] remove debug message

---
 bbot/modules/dnsbrute.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py
index 2df3a48d6..8dc014084 100644
--- a/bbot/modules/dnsbrute.py
+++ b/bbot/modules/dnsbrute.py
@@ -43,7 +43,6 @@ async def eligible_for_enumeration(self, event):
         return eligible, reason
 
     async def handle_event(self, event):
-        self.hugewarning(event)
         query = self.make_query(event)
         self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
         for hostname in await self.helpers.dns.brute(self, query, self.subdomain_list):

From a86c9d4a5d4175ed257e05ca0040868df9011edc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Apr 2024 04:32:52 +0000
Subject: [PATCH 020/220] Bump pydantic from 2.7.0 to 2.7.1

Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.7.0 to 2.7.1.
- [Release notes](https://github.com/pydantic/pydantic/releases)
- [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md)
- [Commits](https://github.com/pydantic/pydantic/compare/v2.7.0...v2.7.1)

---
updated-dependencies:
- dependency-name: pydantic
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 170 ++++++++++++++++++++++++++--------------------------
 1 file changed, 86 insertions(+), 84 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index d47650bf4..f740cc616 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -942,6 +942,7 @@ files = [
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"},
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"},
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"},
+    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"},
@@ -1564,18 +1565,18 @@ files = [
 
 [[package]]
 name = "pydantic"
-version = "2.7.0"
+version = "2.7.1"
 description = "Data validation using Python type hints"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pydantic-2.7.0-py3-none-any.whl", hash = "sha256:9dee74a271705f14f9a1567671d144a851c675b072736f0a7b2608fd9e495352"},
-    {file = "pydantic-2.7.0.tar.gz", hash = "sha256:b5ecdd42262ca2462e2624793551e80911a1e989f462910bb81aef974b4bb383"},
+    {file = "pydantic-2.7.1-py3-none-any.whl", hash = "sha256:e029badca45266732a9a79898a15ae2e8b14840b1eabbb25844be28f0b33f3d5"},
+    {file = "pydantic-2.7.1.tar.gz", hash = "sha256:e9dbb5eada8abe4d9ae5f46b9939aead650cd2b68f249bb3a8139dbe125803cc"},
 ]
 
 [package.dependencies]
 annotated-types = ">=0.4.0"
-pydantic-core = "2.18.1"
+pydantic-core = "2.18.2"
 typing-extensions = ">=4.6.1"
 
 [package.extras]
@@ -1583,90 +1584,90 @@ email = ["email-validator (>=2.0.0)"]
 
 [[package]]
 name = "pydantic-core"
-version = "2.18.1"
+version = "2.18.2"
 description = "Core functionality for Pydantic validation and serialization"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pydantic_core-2.18.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ee9cf33e7fe14243f5ca6977658eb7d1042caaa66847daacbd2117adb258b226"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6b7bbb97d82659ac8b37450c60ff2e9f97e4eb0f8a8a3645a5568b9334b08b50"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df4249b579e75094f7e9bb4bd28231acf55e308bf686b952f43100a5a0be394c"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d0491006a6ad20507aec2be72e7831a42efc93193d2402018007ff827dc62926"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ae80f72bb7a3e397ab37b53a2b49c62cc5496412e71bc4f1277620a7ce3f52b"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58aca931bef83217fca7a390e0486ae327c4af9c3e941adb75f8772f8eeb03a1"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1be91ad664fc9245404a789d60cba1e91c26b1454ba136d2a1bf0c2ac0c0505a"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:667880321e916a8920ef49f5d50e7983792cf59f3b6079f3c9dac2b88a311d17"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f7054fdc556f5421f01e39cbb767d5ec5c1139ea98c3e5b350e02e62201740c7"},
-    {file = "pydantic_core-2.18.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:030e4f9516f9947f38179249778709a460a3adb516bf39b5eb9066fcfe43d0e6"},
-    {file = "pydantic_core-2.18.1-cp310-none-win32.whl", hash = "sha256:2e91711e36e229978d92642bfc3546333a9127ecebb3f2761372e096395fc649"},
-    {file = "pydantic_core-2.18.1-cp310-none-win_amd64.whl", hash = "sha256:9a29726f91c6cb390b3c2338f0df5cd3e216ad7a938762d11c994bb37552edb0"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:9ece8a49696669d483d206b4474c367852c44815fca23ac4e48b72b339807f80"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a5d83efc109ceddb99abd2c1316298ced2adb4570410defe766851a804fcd5b"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f7973c381283783cd1043a8c8f61ea5ce7a3a58b0369f0ee0ee975eaf2f2a1b"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54c7375c62190a7845091f521add19b0f026bcf6ae674bdb89f296972272e86d"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd63cec4e26e790b70544ae5cc48d11b515b09e05fdd5eff12e3195f54b8a586"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:561cf62c8a3498406495cfc49eee086ed2bb186d08bcc65812b75fda42c38294"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68717c38a68e37af87c4da20e08f3e27d7e4212e99e96c3d875fbf3f4812abfc"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d5728e93d28a3c63ee513d9ffbac9c5989de8c76e049dbcb5bfe4b923a9739d"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f0f17814c505f07806e22b28856c59ac80cee7dd0fbb152aed273e116378f519"},
-    {file = "pydantic_core-2.18.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d816f44a51ba5175394bc6c7879ca0bd2be560b2c9e9f3411ef3a4cbe644c2e9"},
-    {file = "pydantic_core-2.18.1-cp311-none-win32.whl", hash = "sha256:09f03dfc0ef8c22622eaa8608caa4a1e189cfb83ce847045eca34f690895eccb"},
-    {file = "pydantic_core-2.18.1-cp311-none-win_amd64.whl", hash = "sha256:27f1009dc292f3b7ca77feb3571c537276b9aad5dd4efb471ac88a8bd09024e9"},
-    {file = "pydantic_core-2.18.1-cp311-none-win_arm64.whl", hash = "sha256:48dd883db92e92519201f2b01cafa881e5f7125666141a49ffba8b9facc072b0"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b6b0e4912030c6f28bcb72b9ebe4989d6dc2eebcd2a9cdc35fefc38052dd4fe8"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3202a429fe825b699c57892d4371c74cc3456d8d71b7f35d6028c96dfecad31"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3982b0a32d0a88b3907e4b0dc36809fda477f0757c59a505d4e9b455f384b8b"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25595ac311f20e5324d1941909b0d12933f1fd2171075fcff763e90f43e92a0d"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14fe73881cf8e4cbdaded8ca0aa671635b597e42447fec7060d0868b52d074e6"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca976884ce34070799e4dfc6fbd68cb1d181db1eefe4a3a94798ddfb34b8867f"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:684d840d2c9ec5de9cb397fcb3f36d5ebb6fa0d94734f9886032dd796c1ead06"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:54764c083bbe0264f0f746cefcded6cb08fbbaaf1ad1d78fb8a4c30cff999a90"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:201713f2f462e5c015b343e86e68bd8a530a4f76609b33d8f0ec65d2b921712a"},
-    {file = "pydantic_core-2.18.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fd1a9edb9dd9d79fbeac1ea1f9a8dd527a6113b18d2e9bcc0d541d308dae639b"},
-    {file = "pydantic_core-2.18.1-cp312-none-win32.whl", hash = "sha256:d5e6b7155b8197b329dc787356cfd2684c9d6a6b1a197f6bbf45f5555a98d411"},
-    {file = "pydantic_core-2.18.1-cp312-none-win_amd64.whl", hash = "sha256:9376d83d686ec62e8b19c0ac3bf8d28d8a5981d0df290196fb6ef24d8a26f0d6"},
-    {file = "pydantic_core-2.18.1-cp312-none-win_arm64.whl", hash = "sha256:c562b49c96906b4029b5685075fe1ebd3b5cc2601dfa0b9e16c2c09d6cbce048"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3e352f0191d99fe617371096845070dee295444979efb8f27ad941227de6ad09"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0295d52b012cbe0d3059b1dba99159c3be55e632aae1999ab74ae2bd86a33d7"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56823a92075780582d1ffd4489a2e61d56fd3ebb4b40b713d63f96dd92d28144"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd3f79e17b56741b5177bcc36307750d50ea0698df6aa82f69c7db32d968c1c2"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38a5024de321d672a132b1834a66eeb7931959c59964b777e8f32dbe9523f6b1"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2ce426ee691319d4767748c8e0895cfc56593d725594e415f274059bcf3cb76"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2adaeea59849ec0939af5c5d476935f2bab4b7f0335b0110f0f069a41024278e"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9b6431559676a1079eac0f52d6d0721fb8e3c5ba43c37bc537c8c83724031feb"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:85233abb44bc18d16e72dc05bf13848a36f363f83757541f1a97db2f8d58cfd9"},
-    {file = "pydantic_core-2.18.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:641a018af4fe48be57a2b3d7a1f0f5dbca07c1d00951d3d7463f0ac9dac66622"},
-    {file = "pydantic_core-2.18.1-cp38-none-win32.whl", hash = "sha256:63d7523cd95d2fde0d28dc42968ac731b5bb1e516cc56b93a50ab293f4daeaad"},
-    {file = "pydantic_core-2.18.1-cp38-none-win_amd64.whl", hash = "sha256:907a4d7720abfcb1c81619863efd47c8a85d26a257a2dbebdb87c3b847df0278"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:aad17e462f42ddbef5984d70c40bfc4146c322a2da79715932cd8976317054de"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:94b9769ba435b598b547c762184bcfc4783d0d4c7771b04a3b45775c3589ca44"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80e0e57cc704a52fb1b48f16d5b2c8818da087dbee6f98d9bf19546930dc64b5"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:76b86e24039c35280ceee6dce7e62945eb93a5175d43689ba98360ab31eebc4a"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12a05db5013ec0ca4a32cc6433f53faa2a014ec364031408540ba858c2172bb0"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:250ae39445cb5475e483a36b1061af1bc233de3e9ad0f4f76a71b66231b07f88"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a32204489259786a923e02990249c65b0f17235073149d0033efcebe80095570"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6395a4435fa26519fd96fdccb77e9d00ddae9dd6c742309bd0b5610609ad7fb2"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2533ad2883f001efa72f3d0e733fb846710c3af6dcdd544fe5bf14fa5fe2d7db"},
-    {file = "pydantic_core-2.18.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b560b72ed4816aee52783c66854d96157fd8175631f01ef58e894cc57c84f0f6"},
-    {file = "pydantic_core-2.18.1-cp39-none-win32.whl", hash = "sha256:582cf2cead97c9e382a7f4d3b744cf0ef1a6e815e44d3aa81af3ad98762f5a9b"},
-    {file = "pydantic_core-2.18.1-cp39-none-win_amd64.whl", hash = "sha256:ca71d501629d1fa50ea7fa3b08ba884fe10cefc559f5c6c8dfe9036c16e8ae89"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e178e5b66a06ec5bf51668ec0d4ac8cfb2bdcb553b2c207d58148340efd00143"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:72722ce529a76a4637a60be18bd789d8fb871e84472490ed7ddff62d5fed620d"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fe0c1ce5b129455e43f941f7a46f61f3d3861e571f2905d55cdbb8b5c6f5e2c"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4284c621f06a72ce2cb55f74ea3150113d926a6eb78ab38340c08f770eb9b4d"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0c3e718f4e064efde68092d9d974e39572c14e56726ecfaeebbe6544521f47"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2027493cc44c23b598cfaf200936110433d9caa84e2c6cf487a83999638a96ac"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:76909849d1a6bffa5a07742294f3fa1d357dc917cb1fe7b470afbc3a7579d539"},
-    {file = "pydantic_core-2.18.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ee7ccc7fb7e921d767f853b47814c3048c7de536663e82fbc37f5eb0d532224b"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ee2794111c188548a4547eccc73a6a8527fe2af6cf25e1a4ebda2fd01cdd2e60"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a139fe9f298dc097349fb4f28c8b81cc7a202dbfba66af0e14be5cfca4ef7ce5"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d074b07a10c391fc5bbdcb37b2f16f20fcd9e51e10d01652ab298c0d07908ee2"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c69567ddbac186e8c0aadc1f324a60a564cfe25e43ef2ce81bcc4b8c3abffbae"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf1c7b78cddb5af00971ad5294a4583188bda1495b13760d9f03c9483bb6203"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2684a94fdfd1b146ff10689c6e4e815f6a01141781c493b97342cdc5b06f4d5d"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:73c1bc8a86a5c9e8721a088df234265317692d0b5cd9e86e975ce3bc3db62a59"},
-    {file = "pydantic_core-2.18.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:e60defc3c15defb70bb38dd605ff7e0fae5f6c9c7cbfe0ad7868582cb7e844a6"},
-    {file = "pydantic_core-2.18.1.tar.gz", hash = "sha256:de9d3e8717560eb05e28739d1b35e4eac2e458553a52a301e51352a7ffc86a35"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9e08e867b306f525802df7cd16c44ff5ebbe747ff0ca6cf3fde7f36c05a59a81"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f0a21cbaa69900cbe1a2e7cad2aa74ac3cf21b10c3efb0fa0b80305274c0e8a2"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0680b1f1f11fda801397de52c36ce38ef1c1dc841a0927a94f226dea29c3ae3d"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:95b9d5e72481d3780ba3442eac863eae92ae43a5f3adb5b4d0a1de89d42bb250"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fcf5cd9c4b655ad666ca332b9a081112cd7a58a8b5a6ca7a3104bc950f2038"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b5155ff768083cb1d62f3e143b49a8a3432e6789a3abee8acd005c3c7af1c74"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:553ef617b6836fc7e4df130bb851e32fe357ce36336d897fd6646d6058d980af"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89ed9eb7d616ef5714e5590e6cf7f23b02d0d539767d33561e3675d6f9e3857"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:75f7e9488238e920ab6204399ded280dc4c307d034f3924cd7f90a38b1829563"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ef26c9e94a8c04a1b2924149a9cb081836913818e55681722d7f29af88fe7b38"},
+    {file = "pydantic_core-2.18.2-cp310-none-win32.whl", hash = "sha256:182245ff6b0039e82b6bb585ed55a64d7c81c560715d1bad0cbad6dfa07b4027"},
+    {file = "pydantic_core-2.18.2-cp310-none-win_amd64.whl", hash = "sha256:e23ec367a948b6d812301afc1b13f8094ab7b2c280af66ef450efc357d2ae543"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:219da3f096d50a157f33645a1cf31c0ad1fe829a92181dd1311022f986e5fbe3"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cc1cfd88a64e012b74e94cd00bbe0f9c6df57049c97f02bb07d39e9c852e19a4"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b7133a6e6aeb8df37d6f413f7705a37ab4031597f64ab56384c94d98fa0e90"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:224c421235f6102e8737032483f43c1a8cfb1d2f45740c44166219599358c2cd"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b14d82cdb934e99dda6d9d60dc84a24379820176cc4a0d123f88df319ae9c150"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2728b01246a3bba6de144f9e3115b532ee44bd6cf39795194fb75491824a1413"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470b94480bb5ee929f5acba6995251ada5e059a5ef3e0dfc63cca287283ebfa6"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:997abc4df705d1295a42f95b4eec4950a37ad8ae46d913caeee117b6b198811c"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75250dbc5290e3f1a0f4618db35e51a165186f9034eff158f3d490b3fed9f8a0"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4456f2dca97c425231d7315737d45239b2b51a50dc2b6f0c2bb181fce6207664"},
+    {file = "pydantic_core-2.18.2-cp311-none-win32.whl", hash = "sha256:269322dcc3d8bdb69f054681edff86276b2ff972447863cf34c8b860f5188e2e"},
+    {file = "pydantic_core-2.18.2-cp311-none-win_amd64.whl", hash = "sha256:800d60565aec896f25bc3cfa56d2277d52d5182af08162f7954f938c06dc4ee3"},
+    {file = "pydantic_core-2.18.2-cp311-none-win_arm64.whl", hash = "sha256:1404c69d6a676245199767ba4f633cce5f4ad4181f9d0ccb0577e1f66cf4c46d"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:fb2bd7be70c0fe4dfd32c951bc813d9fe6ebcbfdd15a07527796c8204bd36242"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6132dd3bd52838acddca05a72aafb6eab6536aa145e923bb50f45e78b7251043"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d904828195733c183d20a54230c0df0eb46ec746ea1a666730787353e87182"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c9bd70772c720142be1020eac55f8143a34ec9f82d75a8e7a07852023e46617f"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8ed04b3582771764538f7ee7001b02e1170223cf9b75dff0bc698fadb00cf3"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6dac87ddb34aaec85f873d737e9d06a3555a1cc1a8e0c44b7f8d5daeb89d86f"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ca4ae5a27ad7a4ee5170aebce1574b375de390bc01284f87b18d43a3984df72"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:886eec03591b7cf058467a70a87733b35f44707bd86cf64a615584fd72488b7c"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ca7b0c1f1c983e064caa85f3792dd2fe3526b3505378874afa84baf662e12241"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b4356d3538c3649337df4074e81b85f0616b79731fe22dd11b99499b2ebbdf3"},
+    {file = "pydantic_core-2.18.2-cp312-none-win32.whl", hash = "sha256:8b172601454f2d7701121bbec3425dd71efcb787a027edf49724c9cefc14c038"},
+    {file = "pydantic_core-2.18.2-cp312-none-win_amd64.whl", hash = "sha256:b1bd7e47b1558ea872bd16c8502c414f9e90dcf12f1395129d7bb42a09a95438"},
+    {file = "pydantic_core-2.18.2-cp312-none-win_arm64.whl", hash = "sha256:98758d627ff397e752bc339272c14c98199c613f922d4a384ddc07526c86a2ec"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9fdad8e35f278b2c3eb77cbdc5c0a49dada440657bf738d6905ce106dc1de439"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1d90c3265ae107f91a4f279f4d6f6f1d4907ac76c6868b27dc7fb33688cfb347"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:390193c770399861d8df9670fb0d1874f330c79caaca4642332df7c682bf6b91"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:82d5d4d78e4448683cb467897fe24e2b74bb7b973a541ea1dcfec1d3cbce39fb"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4774f3184d2ef3e14e8693194f661dea5a4d6ca4e3dc8e39786d33a94865cefd"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4d938ec0adf5167cb335acb25a4ee69a8107e4984f8fbd2e897021d9e4ca21b"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0e8b1be28239fc64a88a8189d1df7fad8be8c1ae47fcc33e43d4be15f99cc70"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:868649da93e5a3d5eacc2b5b3b9235c98ccdbfd443832f31e075f54419e1b96b"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:78363590ef93d5d226ba21a90a03ea89a20738ee5b7da83d771d283fd8a56761"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:852e966fbd035a6468fc0a3496589b45e2208ec7ca95c26470a54daed82a0788"},
+    {file = "pydantic_core-2.18.2-cp38-none-win32.whl", hash = "sha256:6a46e22a707e7ad4484ac9ee9f290f9d501df45954184e23fc29408dfad61350"},
+    {file = "pydantic_core-2.18.2-cp38-none-win_amd64.whl", hash = "sha256:d91cb5ea8b11607cc757675051f61b3d93f15eca3cefb3e6c704a5d6e8440f4e"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ae0a8a797a5e56c053610fa7be147993fe50960fa43609ff2a9552b0e07013e8"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:042473b6280246b1dbf530559246f6842b56119c2926d1e52b631bdc46075f2a"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a388a77e629b9ec814c1b1e6b3b595fe521d2cdc625fcca26fbc2d44c816804"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25add29b8f3b233ae90ccef2d902d0ae0432eb0d45370fe315d1a5cf231004b"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f459a5ce8434614dfd39bbebf1041952ae01da6bed9855008cb33b875cb024c0"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eff2de745698eb46eeb51193a9f41d67d834d50e424aef27df2fcdee1b153845"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8309f67285bdfe65c372ea3722b7a5642680f3dba538566340a9d36e920b5f0"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f93a8a2e3938ff656a7c1bc57193b1319960ac015b6e87d76c76bf14fe0244b4"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:22057013c8c1e272eb8d0eebc796701167d8377441ec894a8fed1af64a0bf399"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cfeecd1ac6cc1fb2692c3d5110781c965aabd4ec5d32799773ca7b1456ac636b"},
+    {file = "pydantic_core-2.18.2-cp39-none-win32.whl", hash = "sha256:0d69b4c2f6bb3e130dba60d34c0845ba31b69babdd3f78f7c0c8fae5021a253e"},
+    {file = "pydantic_core-2.18.2-cp39-none-win_amd64.whl", hash = "sha256:d9319e499827271b09b4e411905b24a426b8fb69464dfa1696258f53a3334641"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a1874c6dd4113308bd0eb568418e6114b252afe44319ead2b4081e9b9521fe75"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:ccdd111c03bfd3666bd2472b674c6899550e09e9f298954cfc896ab92b5b0e6d"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e18609ceaa6eed63753037fc06ebb16041d17d28199ae5aba0052c51449650a9"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e5c584d357c4e2baf0ff7baf44f4994be121e16a2c88918a5817331fc7599d7"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43f0f463cf89ace478de71a318b1b4f05ebc456a9b9300d027b4b57c1a2064fb"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e1b395e58b10b73b07b7cf740d728dd4ff9365ac46c18751bf8b3d8cca8f625a"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0098300eebb1c837271d3d1a2cd2911e7c11b396eac9661655ee524a7f10587b"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:36789b70d613fbac0a25bb07ab3d9dba4d2e38af609c020cf4d888d165ee0bf3"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3f9a801e7c8f1ef8718da265bba008fa121243dfe37c1cea17840b0944dfd72c"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:3a6515ebc6e69d85502b4951d89131ca4e036078ea35533bb76327f8424531ce"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20aca1e2298c56ececfd8ed159ae4dde2df0781988c97ef77d5c16ff4bd5b400"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:223ee893d77a310a0391dca6df00f70bbc2f36a71a895cecd9a0e762dc37b349"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2334ce8c673ee93a1d6a65bd90327588387ba073c17e61bf19b4fd97d688d63c"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cbca948f2d14b09d20268cda7b0367723d79063f26c4ffc523af9042cad95592"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b3ef08e20ec49e02d5c6717a91bb5af9b20f1805583cb0adfe9ba2c6b505b5ae"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6fdc8627910eed0c01aed6a390a252fe3ea6d472ee70fdde56273f198938374"},
+    {file = "pydantic_core-2.18.2.tar.gz", hash = "sha256:2e29d20810dfc3043ee13ac7d9e25105799817683348823f305ab3f349b9386e"},
 ]
 
 [package.dependencies]
@@ -2109,6 +2110,7 @@ optional = false
 python-versions = "*"
 files = [
     {file = "requests-file-2.0.0.tar.gz", hash = "sha256:20c5931629c558fda566cacc10cfe2cd502433e628f568c34c80d96a0cc95972"},
+    {file = "requests_file-2.0.0-py2.py3-none-any.whl", hash = "sha256:3e493d390adb44aa102ebea827a48717336d5268968c370eaf19abaf5cae13bf"},
 ]
 
 [package.dependencies]

From 24a2fd3b91d253092ed0d13de6fcd784436995f0 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 08:22:19 -0400
Subject: [PATCH 021/220] remove DNS_NAME_UNRESOLVED exception for targets

---
 bbot/modules/internal/dns.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index b96b9b19c..29e060626 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -164,7 +164,7 @@ async def handle_event(self, event, kwargs):
                 event.add_tag(tag)
 
         # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
-        if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags:
+        if event.type == "DNS_NAME" and "unresolved" in event.tags:
             event.type = "DNS_NAME_UNRESOLVED"
 
         # speculate DNS_NAMES and IP_ADDRESSes from other event types

From bc0786a6b01e11ddbf46d174f588fe816556edc7 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 08:35:01 -0400
Subject: [PATCH 022/220] always emit targets

---
 bbot/modules/output/base.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index 9459a880f..d0365f079 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -33,10 +33,14 @@ def _event_precheck(self, event):
         if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
             return False, "its extension was listed in url_extension_httpx_only"
 
-        # output module specific stuff
-        # omitted events such as HTTP_RESPONSE etc.
-        if event._omit and not event.type in self.get_watched_events():
-            return False, "_omit is True"
+        ### begin output-module specific ###
+
+        # events from omit_event_types, e.g. HTTP_RESPONSE, DNS_NAME_UNRESOLVED, etc.
+        # if the output module specifically requests a certain event type, we let it through anyway
+        # always_emit overrides _omit.
+        if event._omit:
+            if not event.always_emit and not event.type in self.get_watched_events():
+                return False, "_omit is True"
 
         # force-output certain events to the graph
         if self._is_graph_important(event):

From 0301442ff314c1f2bec451fa4b79b8b61220b238 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 08:37:09 -0400
Subject: [PATCH 023/220] targets only

---
 bbot/modules/output/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index d0365f079..c9a41eebd 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -37,9 +37,9 @@ def _event_precheck(self, event):
 
         # events from omit_event_types, e.g. HTTP_RESPONSE, DNS_NAME_UNRESOLVED, etc.
         # if the output module specifically requests a certain event type, we let it through anyway
-        # always_emit overrides _omit.
+        # an exception is also made for targets
         if event._omit:
-            if not event.always_emit and not event.type in self.get_watched_events():
+            if not "target" in event.tags and not event.type in self.get_watched_events():
                 return False, "_omit is True"
 
         # force-output certain events to the graph

From c53d57875644a6e9aad1658f3b896448cb1e7918 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Mon, 29 Apr 2024 16:42:20 +0100
Subject: [PATCH 024/220] Inital add

---
 bbot/modules/github_workflows.py              | 144 +++++
 .../test_module_github_workflows.py           | 573 ++++++++++++++++++
 2 files changed, 717 insertions(+)
 create mode 100644 bbot/modules/github_workflows.py
 create mode 100644 bbot/test/test_step_2/module_tests/test_module_github_workflows.py

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
new file mode 100644
index 000000000..5719153c2
--- /dev/null
+++ b/bbot/modules/github_workflows.py
@@ -0,0 +1,144 @@
+from bbot.modules.templates.github import github
+
+
+class github_workflows(github):
+    watched_events = ["CODE_REPOSITORY"]
+    produced_events = ["HTTP_RESPONSE"]
+    flags = ["passive", "safe"]
+    meta = {"description": "Query Github's API for the repositories workflow logs"}
+    options = {"api_key": ""}
+    options_desc = {
+        "api_key": "Github token",
+    }
+
+    # scope_distance_modifier = 2
+
+    async def setup(self):
+        return await super().setup()
+
+    async def filter_event(self, event):
+        if event.type == "CODE_REPOSITORY":
+            if "git" not in event.tags:
+                return False, "event is not a git repository"
+        return True
+
+    async def handle_event(self, event):
+        # repo_url = event.data.get("url")
+        owner = "blacklanternsecurity"
+        repo = "bbot"
+        for workflow in await self.get_workflows(owner, repo):
+            workflow_name = workflow.get("name")
+            workflow_id = workflow.get("id")
+            self.log.debug(f"Looking up runs for {workflow_name} in {owner}/{repo}")
+            for run in await self.get_workflow_runs(owner, repo, workflow_id):
+                run_id = run.get("id")
+                self.log.debug(f"Looking up jobs for {workflow_name}/{run_id} in {owner}/{repo}")
+                for job in await self.get_run_jobs(owner, repo, run_id):
+                    job_id = job.get("id")
+                    commit_id = job.get("head_sha")
+                    steps = job.get("steps", [])
+                    for step in steps:
+                        if step.get("conclusion") == "success":
+                            step_name = step.get("name")
+                            number = step.get("number")
+                            self.log.debug(
+                                f"Requesting {workflow_name}/run {run_id}/job {job_id}/{step_name} log for {owner}/{repo}"
+                            )
+                            # Request log step from the html_url as that bypasses the admin restrictions from using the API
+                            response = await self.helpers.request(
+                                f"https://github.com/{owner}/{repo}/commit/{commit_id}/checks/{job_id}/logs/{number}",
+                                follow_redirects=True,
+                            )
+                            if response:
+                                blob_url = response.headers.get("Location", "")
+                                if blob_url:
+                                    url_event = self.make_event(
+                                        blob_url, "URL_UNVERIFIED", source=event, tags=["httpx-safe"]
+                                    )
+                                    if not url_event:
+                                        continue
+                                    url_event.scope_distance = event.scope_distance
+                                    await self.emit_event(url_event)
+
+    async def get_workflows(self, owner, repo):
+        workflows = []
+        url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows?per_page=100&page=" + "{page}"
+        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        try:
+            async for r in agen:
+                if r is None:
+                    break
+                status_code = getattr(r, "status_code", 0)
+                if status_code == 403:
+                    self.warning("Github is rate-limiting us (HTTP status: 403)")
+                    break
+                if status_code != 200:
+                    break
+                try:
+                    j = r.json()
+                except Exception as e:
+                    self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+                    break
+                if not j:
+                    break
+                for item in j.get("workflows", []):
+                    workflows.append(item)
+        finally:
+            agen.aclose()
+        return workflows
+
+    async def get_workflow_runs(self, owner, repo, workflow_id):
+        runs = []
+        url = (
+            f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?per_page=100&page=" + "{page}"
+        )
+        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        try:
+            async for r in agen:
+                if r is None:
+                    break
+                status_code = getattr(r, "status_code", 0)
+                if status_code == 403:
+                    self.warning("Github is rate-limiting us (HTTP status: 403)")
+                    break
+                if status_code != 200:
+                    break
+                try:
+                    j = r.json()
+                except Exception as e:
+                    self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+                    break
+                if not j:
+                    break
+                for item in j.get("workflow_runs", []):
+                    runs.append(item)
+        finally:
+            agen.aclose()
+        return runs
+
+    async def get_run_jobs(self, owner, repo, run_id):
+        jobs = []
+        url = f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/jobs?per_page=100&page=" + "{page}"
+        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        try:
+            async for r in agen:
+                if r is None:
+                    break
+                status_code = getattr(r, "status_code", 0)
+                if status_code == 403:
+                    self.warning("Github is rate-limiting us (HTTP status: 403)")
+                    break
+                if status_code != 200:
+                    break
+                try:
+                    j = r.json()
+                except Exception as e:
+                    self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+                    break
+                if not j:
+                    break
+                for item in j.get("jobs", []):
+                    jobs.append(item)
+        finally:
+            agen.aclose()
+        return jobs
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
new file mode 100644
index 000000000..ee2f93a49
--- /dev/null
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -0,0 +1,573 @@
+from .base import ModuleTestBase
+
+
+class TestGithub_Workflows(ModuleTestBase):
+    config_overrides = {"modules": {"github_org": {"api_key": "asdf"}}}
+    modules_overrides = ["github_workflows", "github_org", "speculate"]
+
+    async def setup_before_prep(self, module_test):
+        module_test.httpx_mock.add_response(url="https://api.github.com/zen")
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/orgs/blacklanternsecurity",
+            json={
+                "login": "blacklanternsecurity",
+                "id": 25311592,
+                "node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky",
+                "url": "https://api.github.com/orgs/blacklanternsecurity",
+                "repos_url": "https://api.github.com/orgs/blacklanternsecurity/repos",
+                "events_url": "https://api.github.com/orgs/blacklanternsecurity/events",
+                "hooks_url": "https://api.github.com/orgs/blacklanternsecurity/hooks",
+                "issues_url": "https://api.github.com/orgs/blacklanternsecurity/issues",
+                "members_url": "https://api.github.com/orgs/blacklanternsecurity/members{/member}",
+                "public_members_url": "https://api.github.com/orgs/blacklanternsecurity/public_members{/member}",
+                "avatar_url": "https://avatars.githubusercontent.com/u/25311592?v=4",
+                "description": "Security Organization",
+                "name": "Black Lantern Security",
+                "company": None,
+                "blog": "www.blacklanternsecurity.com",
+                "location": "Charleston, SC",
+                "email": None,
+                "twitter_username": None,
+                "is_verified": False,
+                "has_organization_projects": True,
+                "has_repository_projects": True,
+                "public_repos": 70,
+                "public_gists": 0,
+                "followers": 415,
+                "following": 0,
+                "html_url": "https://github.com/blacklanternsecurity",
+                "created_at": "2017-01-24T00:14:46Z",
+                "updated_at": "2022-03-28T11:39:03Z",
+                "archived_at": None,
+                "type": "Organization",
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/orgs/blacklanternsecurity/repos?per_page=100&page=1",
+            json=[
+                {
+                    "id": 459780477,
+                    "node_id": "R_kgDOG2exfQ",
+                    "name": "test_keys",
+                    "full_name": "blacklanternsecurity/test_keys",
+                    "private": False,
+                    "owner": {
+                        "login": "blacklanternsecurity",
+                        "id": 79229934,
+                        "node_id": "MDEyOk9yZ2FuaXphdGlvbjc5MjI5OTM0",
+                        "avatar_url": "https://avatars.githubusercontent.com/u/79229934?v=4",
+                        "gravatar_id": "",
+                        "url": "https://api.github.com/users/blacklanternsecurity",
+                        "html_url": "https://github.com/blacklanternsecurity",
+                        "followers_url": "https://api.github.com/users/blacklanternsecurity/followers",
+                        "following_url": "https://api.github.com/users/blacklanternsecurity/following{/other_user}",
+                        "gists_url": "https://api.github.com/users/blacklanternsecurity/gists{/gist_id}",
+                        "starred_url": "https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}",
+                        "subscriptions_url": "https://api.github.com/users/blacklanternsecurity/subscriptions",
+                        "organizations_url": "https://api.github.com/users/blacklanternsecurity/orgs",
+                        "repos_url": "https://api.github.com/users/blacklanternsecurity/repos",
+                        "events_url": "https://api.github.com/users/blacklanternsecurity/events{/privacy}",
+                        "received_events_url": "https://api.github.com/users/blacklanternsecurity/received_events",
+                        "type": "Organization",
+                        "site_admin": False,
+                    },
+                    "html_url": "https://github.com/blacklanternsecurity/test_keys",
+                    "description": None,
+                    "fork": False,
+                    "url": "https://api.github.com/repos/blacklanternsecurity/test_keys",
+                    "forks_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/forks",
+                    "keys_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/keys{/key_id}",
+                    "collaborators_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/collaborators{/collaborator}",
+                    "teams_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/teams",
+                    "hooks_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/hooks",
+                    "issue_events_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/issues/events{/number}",
+                    "events_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/events",
+                    "assignees_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/assignees{/user}",
+                    "branches_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/branches{/branch}",
+                    "tags_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/tags",
+                    "blobs_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/git/blobs{/sha}",
+                    "git_tags_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/git/tags{/sha}",
+                    "git_refs_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/git/refs{/sha}",
+                    "trees_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/git/trees{/sha}",
+                    "statuses_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/statuses/{sha}",
+                    "languages_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/languages",
+                    "stargazers_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/stargazers",
+                    "contributors_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/contributors",
+                    "subscribers_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/subscribers",
+                    "subscription_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/subscription",
+                    "commits_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/commits{/sha}",
+                    "git_commits_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/git/commits{/sha}",
+                    "comments_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/comments{/number}",
+                    "issue_comment_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/issues/comments{/number}",
+                    "contents_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/contents/{+path}",
+                    "compare_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/compare/{base}...{head}",
+                    "merges_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/merges",
+                    "archive_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/{archive_format}{/ref}",
+                    "downloads_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/downloads",
+                    "issues_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/issues{/number}",
+                    "pulls_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/pulls{/number}",
+                    "milestones_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/milestones{/number}",
+                    "notifications_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/notifications{?since,all,participating}",
+                    "labels_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/labels{/name}",
+                    "releases_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/releases{/id}",
+                    "deployments_url": "https://api.github.com/repos/blacklanternsecurity/test_keys/deployments",
+                    "created_at": "2022-02-15T23:10:51Z",
+                    "updated_at": "2023-09-02T12:20:13Z",
+                    "pushed_at": "2023-10-19T02:56:46Z",
+                    "git_url": "git://github.com/blacklanternsecurity/test_keys.git",
+                    "ssh_url": "git@github.com:blacklanternsecurity/test_keys.git",
+                    "clone_url": "https://github.com/blacklanternsecurity/test_keys.git",
+                    "svn_url": "https://github.com/blacklanternsecurity/test_keys",
+                    "homepage": None,
+                    "size": 2,
+                    "stargazers_count": 2,
+                    "watchers_count": 2,
+                    "language": None,
+                    "has_issues": True,
+                    "has_projects": True,
+                    "has_downloads": True,
+                    "has_wiki": True,
+                    "has_pages": False,
+                    "has_discussions": False,
+                    "forks_count": 32,
+                    "mirror_url": None,
+                    "archived": False,
+                    "disabled": False,
+                    "open_issues_count": 2,
+                    "license": None,
+                    "allow_forking": True,
+                    "is_template": False,
+                    "web_commit_signoff_required": False,
+                    "topics": [],
+                    "visibility": "public",
+                    "forks": 32,
+                    "open_issues": 2,
+                    "watchers": 2,
+                    "default_branch": "main",
+                    "permissions": {"admin": False, "maintain": False, "push": False, "triage": False, "pull": True},
+                }
+            ],
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows?per_page=100&page=1",
+            json={
+                "total_count": 3,
+                "workflows": [
+                    {
+                        "id": 22452226,
+                        "node_id": "W_kwDOG_O3ns4BVpgC",
+                        "name": "tests",
+                        "path": ".github/workflows/tests.yml",
+                        "state": "active",
+                        "created_at": "2022-03-23T15:09:22.000Z",
+                        "updated_at": "2022-09-27T17:49:34.000Z",
+                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226",
+                        "html_url": "https://github.com/blacklanternsecurity/bbot/blob/stable/.github/workflows/tests.yml",
+                        "badge_url": "https://github.com/blacklanternsecurity/bbot/workflows/tests/badge.svg",
+                    },
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?per_page=100&page=1",
+            json={
+                "total_count": 2993,
+                "workflow_runs": [
+                    {
+                        "id": 8839360698,
+                        "name": "tests",
+                        "node_id": "WFR_kwLOG_O3ns8AAAACDt3wug",
+                        "head_branch": "dnsbrute-helperify",
+                        "head_sha": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                        "path": ".github/workflows/tests.yml",
+                        "display_title": "Helperify Massdns",
+                        "run_number": 4520,
+                        "event": "pull_request",
+                        "status": "completed",
+                        "conclusion": "failure",
+                        "workflow_id": 22452226,
+                        "check_suite_id": 23162098295,
+                        "check_suite_node_id": "CS_kwDOG_O3ns8AAAAFZJGSdw",
+                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698",
+                        "html_url": "https://github.com/blacklanternsecurity/bbot/actions/runs/8839360698",
+                        "pull_requests": [
+                            {
+                                "url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls/1303",
+                                "id": 1839332952,
+                                "number": 1303,
+                                "head": {
+                                    "ref": "dnsbrute-helperify",
+                                    "sha": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                                    "repo": {
+                                        "id": 468957086,
+                                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                                        "name": "bbot",
+                                    },
+                                },
+                                "base": {
+                                    "ref": "faster-regexes",
+                                    "sha": "7baf219c7f3a4ba165639c5ddb62322453a8aea8",
+                                    "repo": {
+                                        "id": 468957086,
+                                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                                        "name": "bbot",
+                                    },
+                                },
+                            }
+                        ],
+                        "created_at": "2024-04-25T21:04:32Z",
+                        "updated_at": "2024-04-25T21:19:43Z",
+                        "actor": {
+                            "login": "TheTechromancer",
+                            "id": 20261699,
+                            "node_id": "MDQ6VXNlcjIwMjYxNjk5",
+                            "avatar_url": "https://avatars.githubusercontent.com/u/20261699?v=4",
+                            "gravatar_id": "",
+                            "url": "https://api.github.com/users/TheTechromancer",
+                            "html_url": "https://github.com/TheTechromancer",
+                            "followers_url": "https://api.github.com/users/TheTechromancer/followers",
+                            "following_url": "https://api.github.com/users/TheTechromancer/following{/other_user}",
+                            "gists_url": "https://api.github.com/users/TheTechromancer/gists{/gist_id}",
+                            "starred_url": "https://api.github.com/users/TheTechromancer/starred{/owner}{/repo}",
+                            "subscriptions_url": "https://api.github.com/users/TheTechromancer/subscriptions",
+                            "organizations_url": "https://api.github.com/users/TheTechromancer/orgs",
+                            "repos_url": "https://api.github.com/users/TheTechromancer/repos",
+                            "events_url": "https://api.github.com/users/TheTechromancer/events{/privacy}",
+                            "received_events_url": "https://api.github.com/users/TheTechromancer/received_events",
+                            "type": "User",
+                            "site_admin": False,
+                        },
+                        "run_attempt": 1,
+                        "referenced_workflows": [],
+                        "run_started_at": "2024-04-25T21:04:32Z",
+                        "triggering_actor": {
+                            "login": "TheTechromancer",
+                            "id": 20261699,
+                            "node_id": "MDQ6VXNlcjIwMjYxNjk5",
+                            "avatar_url": "https://avatars.githubusercontent.com/u/20261699?v=4",
+                            "gravatar_id": "",
+                            "url": "https://api.github.com/users/TheTechromancer",
+                            "html_url": "https://github.com/TheTechromancer",
+                            "followers_url": "https://api.github.com/users/TheTechromancer/followers",
+                            "following_url": "https://api.github.com/users/TheTechromancer/following{/other_user}",
+                            "gists_url": "https://api.github.com/users/TheTechromancer/gists{/gist_id}",
+                            "starred_url": "https://api.github.com/users/TheTechromancer/starred{/owner}{/repo}",
+                            "subscriptions_url": "https://api.github.com/users/TheTechromancer/subscriptions",
+                            "organizations_url": "https://api.github.com/users/TheTechromancer/orgs",
+                            "repos_url": "https://api.github.com/users/TheTechromancer/repos",
+                            "events_url": "https://api.github.com/users/TheTechromancer/events{/privacy}",
+                            "received_events_url": "https://api.github.com/users/TheTechromancer/received_events",
+                            "type": "User",
+                            "site_admin": False,
+                        },
+                        "jobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/jobs",
+                        "logs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/logs",
+                        "check_suite_url": "https://api.github.com/repos/blacklanternsecurity/bbot/check-suites/23162098295",
+                        "artifacts_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/artifacts",
+                        "cancel_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/cancel",
+                        "rerun_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/rerun",
+                        "previous_attempt_url": None,
+                        "workflow_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226",
+                        "head_commit": {
+                            "id": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                            "tree_id": "fe9b345c0745a5bbacb806225e92e1c48fccf35c",
+                            "message": "remove debug message",
+                            "timestamp": "2024-04-25T21:02:37Z",
+                            "author": {"name": "TheTechromancer", "email": "thetechromancer@protonmail.com"},
+                            "committer": {"name": "TheTechromancer", "email": "thetechromancer@protonmail.com"},
+                        },
+                        "repository": {
+                            "id": 468957086,
+                            "node_id": "R_kgDOG_O3ng",
+                            "name": "bbot",
+                            "full_name": "blacklanternsecurity/bbot",
+                            "private": False,
+                            "owner": {
+                                "login": "blacklanternsecurity",
+                                "id": 25311592,
+                                "node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky",
+                                "avatar_url": "https://avatars.githubusercontent.com/u/25311592?v=4",
+                                "gravatar_id": "",
+                                "url": "https://api.github.com/users/blacklanternsecurity",
+                                "html_url": "https://github.com/blacklanternsecurity",
+                                "followers_url": "https://api.github.com/users/blacklanternsecurity/followers",
+                                "following_url": "https://api.github.com/users/blacklanternsecurity/following{/other_user}",
+                                "gists_url": "https://api.github.com/users/blacklanternsecurity/gists{/gist_id}",
+                                "starred_url": "https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}",
+                                "subscriptions_url": "https://api.github.com/users/blacklanternsecurity/subscriptions",
+                                "organizations_url": "https://api.github.com/users/blacklanternsecurity/orgs",
+                                "repos_url": "https://api.github.com/users/blacklanternsecurity/repos",
+                                "events_url": "https://api.github.com/users/blacklanternsecurity/events{/privacy}",
+                                "received_events_url": "https://api.github.com/users/blacklanternsecurity/received_events",
+                                "type": "Organization",
+                                "site_admin": False,
+                            },
+                            "html_url": "https://github.com/blacklanternsecurity/bbot",
+                            "description": "A recursive internet scanner for hackers.",
+                            "fork": False,
+                            "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                            "forks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/forks",
+                            "keys_url": "https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}",
+                            "collaborators_url": "https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}",
+                            "teams_url": "https://api.github.com/repos/blacklanternsecurity/bbot/teams",
+                            "hooks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/hooks",
+                            "issue_events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}",
+                            "events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/events",
+                            "assignees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}",
+                            "branches_url": "https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}",
+                            "tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/tags",
+                            "blobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}",
+                            "git_tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}",
+                            "git_refs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}",
+                            "trees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}",
+                            "statuses_url": "https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}",
+                            "languages_url": "https://api.github.com/repos/blacklanternsecurity/bbot/languages",
+                            "stargazers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/stargazers",
+                            "contributors_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contributors",
+                            "subscribers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscribers",
+                            "subscription_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscription",
+                            "commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}",
+                            "git_commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}",
+                            "comments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}",
+                            "issue_comment_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}",
+                            "contents_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}",
+                            "compare_url": "https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}",
+                            "merges_url": "https://api.github.com/repos/blacklanternsecurity/bbot/merges",
+                            "archive_url": "https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}",
+                            "downloads_url": "https://api.github.com/repos/blacklanternsecurity/bbot/downloads",
+                            "issues_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}",
+                            "pulls_url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}",
+                            "milestones_url": "https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}",
+                            "notifications_url": "https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}",
+                            "labels_url": "https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}",
+                            "releases_url": "https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}",
+                            "deployments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/deployments",
+                        },
+                        "head_repository": {
+                            "id": 468957086,
+                            "node_id": "R_kgDOG_O3ng",
+                            "name": "bbot",
+                            "full_name": "blacklanternsecurity/bbot",
+                            "private": False,
+                            "owner": {
+                                "login": "blacklanternsecurity",
+                                "id": 25311592,
+                                "node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky",
+                                "avatar_url": "https://avatars.githubusercontent.com/u/25311592?v=4",
+                                "gravatar_id": "",
+                                "url": "https://api.github.com/users/blacklanternsecurity",
+                                "html_url": "https://github.com/blacklanternsecurity",
+                                "followers_url": "https://api.github.com/users/blacklanternsecurity/followers",
+                                "following_url": "https://api.github.com/users/blacklanternsecurity/following{/other_user}",
+                                "gists_url": "https://api.github.com/users/blacklanternsecurity/gists{/gist_id}",
+                                "starred_url": "https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}",
+                                "subscriptions_url": "https://api.github.com/users/blacklanternsecurity/subscriptions",
+                                "organizations_url": "https://api.github.com/users/blacklanternsecurity/orgs",
+                                "repos_url": "https://api.github.com/users/blacklanternsecurity/repos",
+                                "events_url": "https://api.github.com/users/blacklanternsecurity/events{/privacy}",
+                                "received_events_url": "https://api.github.com/users/blacklanternsecurity/received_events",
+                                "type": "Organization",
+                                "site_admin": False,
+                            },
+                            "html_url": "https://github.com/blacklanternsecurity/bbot",
+                            "description": "A recursive internet scanner for hackers.",
+                            "fork": False,
+                            "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                            "forks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/forks",
+                            "keys_url": "https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}",
+                            "collaborators_url": "https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}",
+                            "teams_url": "https://api.github.com/repos/blacklanternsecurity/bbot/teams",
+                            "hooks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/hooks",
+                            "issue_events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}",
+                            "events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/events",
+                            "assignees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}",
+                            "branches_url": "https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}",
+                            "tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/tags",
+                            "blobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}",
+                            "git_tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}",
+                            "git_refs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}",
+                            "trees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}",
+                            "statuses_url": "https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}",
+                            "languages_url": "https://api.github.com/repos/blacklanternsecurity/bbot/languages",
+                            "stargazers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/stargazers",
+                            "contributors_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contributors",
+                            "subscribers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscribers",
+                            "subscription_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscription",
+                            "commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}",
+                            "git_commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}",
+                            "comments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}",
+                            "issue_comment_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}",
+                            "contents_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}",
+                            "compare_url": "https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}",
+                            "merges_url": "https://api.github.com/repos/blacklanternsecurity/bbot/merges",
+                            "archive_url": "https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}",
+                            "downloads_url": "https://api.github.com/repos/blacklanternsecurity/bbot/downloads",
+                            "issues_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}",
+                            "pulls_url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}",
+                            "milestones_url": "https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}",
+                            "notifications_url": "https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}",
+                            "labels_url": "https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}",
+                            "releases_url": "https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}",
+                            "deployments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/deployments",
+                        },
+                    },
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/jobs?per_page=100&page=1",
+            json={
+                "total_count": 8,
+                "jobs": [
+                    {
+                        "id": 24272553740,
+                        "run_id": 8839360698,
+                        "workflow_name": "tests",
+                        "head_branch": "dnsbrute-helperify",
+                        "run_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698",
+                        "run_attempt": 1,
+                        "node_id": "CR_kwDOG_O3ns8AAAAFpsHHDA",
+                        "head_sha": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/jobs/24272553740",
+                        "html_url": "https://github.com/blacklanternsecurity/bbot/actions/runs/8839360698/job/24272553740",
+                        "status": "completed",
+                        "conclusion": "success",
+                        "created_at": "2024-04-25T21:04:54Z",
+                        "started_at": "2024-04-25T21:05:01Z",
+                        "completed_at": "2024-04-25T21:05:18Z",
+                        "name": "lint",
+                        "steps": [
+                            {
+                                "name": "Set up job",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 1,
+                                "started_at": "2024-04-25T21:05:00.000Z",
+                                "completed_at": "2024-04-25T21:05:02.000Z",
+                            },
+                            {
+                                "name": "Run actions/checkout@v3",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 2,
+                                "started_at": "2024-04-25T21:05:02.000Z",
+                                "completed_at": "2024-04-25T21:05:04.000Z",
+                            },
+                            {
+                                "name": "Run psf/black@stable",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 3,
+                                "started_at": "2024-04-25T21:05:04.000Z",
+                                "completed_at": "2024-04-25T21:05:10.000Z",
+                            },
+                            {
+                                "name": "Install Python 3",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 4,
+                                "started_at": "2024-04-25T21:05:10.000Z",
+                                "completed_at": "2024-04-25T21:05:11.000Z",
+                            },
+                            {
+                                "name": "Install dependencies",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 5,
+                                "started_at": "2024-04-25T21:05:11.000Z",
+                                "completed_at": "2024-04-25T21:05:12.000Z",
+                            },
+                            {
+                                "name": "flake8",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 6,
+                                "started_at": "2024-04-25T21:05:12.000Z",
+                                "completed_at": "2024-04-25T21:05:14.000Z",
+                            },
+                            {
+                                "name": "Post Install Python 3",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 11,
+                                "started_at": "2024-04-25T21:05:15.000Z",
+                                "completed_at": "2024-04-25T21:05:15.000Z",
+                            },
+                            {
+                                "name": "Post Run actions/checkout@v3",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 12,
+                                "started_at": "2024-04-25T21:05:15.000Z",
+                                "completed_at": "2024-04-25T21:05:15.000Z",
+                            },
+                            {
+                                "name": "Complete job",
+                                "status": "completed",
+                                "conclusion": "success",
+                                "number": 13,
+                                "started_at": "2024-04-25T21:05:15.000Z",
+                                "completed_at": "2024-04-25T21:05:15.000Z",
+                            },
+                        ],
+                        "check_run_url": "https://api.github.com/repos/blacklanternsecurity/bbot/check-runs/24272553740",
+                        "labels": ["ubuntu-latest"],
+                        "runner_id": 60,
+                        "runner_name": "GitHub Actions 60",
+                        "runner_group_id": 2,
+                        "runner_group_name": "GitHub Actions",
+                    },
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://github.com/blacklanternsecurity/bbot/commit/c5de1360e8e5ccba04b23035f675a529282b7dc2/checks/24272553740/logs/1",
+            headers={
+                "location": "https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02"
+            },
+            status_code=302,
+        )
+
+    def check(self, module_test, events):
+        assert len(events) == 5
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com" and e.scope_distance == 0
+            ]
+        ), "Failed to emit target DNS_NAME"
+        assert 1 == len(
+            [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 1]
+        ), "Failed to find ORG_STUB"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "github"
+                and e.data["profile_name"] == "blacklanternsecurity"
+                and "github-org" in e.tags
+                and e.scope_distance == 1
+            ]
+        ), "Failed to find blacklanternsecurity github"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://github.com/blacklanternsecurity/test_keys"
+                and e.scope_distance == 1
+            ]
+        ), "Failed to find blacklanternsecurity github repo"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "URL_UNVERIFIED"
+                and e.data
+                == "https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02"
+                and e.scope_distance == 1
+            ]
+        ), "Failed to obtain redirect to the blob"

From 9b944542a7a001f9b549e40a5201f8d81d5025d6 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Mon, 29 Apr 2024 18:03:32 +0100
Subject: [PATCH 025/220] Changed produced event to `FILESYSTEM`

---
 bbot/modules/github_workflows.py              |  90 +++++--------
 .../test_module_github_workflows.py           | 118 ++----------------
 2 files changed, 41 insertions(+), 167 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 5719153c2..08ca162c5 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -1,19 +1,25 @@
+from datetime import date, timedelta
+
 from bbot.modules.templates.github import github
 
 
 class github_workflows(github):
     watched_events = ["CODE_REPOSITORY"]
-    produced_events = ["HTTP_RESPONSE"]
+    produced_events = ["FILESYSTEM"]
     flags = ["passive", "safe"]
-    meta = {"description": "Query Github's API for the repositories workflow logs"}
-    options = {"api_key": ""}
+    meta = {"description": "Download a github repositories workflow logs"}
+    options = {"api_key": "", "historical_logs": 7}
     options_desc = {
         "api_key": "Github token",
+        "historical_logs": "Fetch logs that are at most this many days old (default: 7)",
     }
 
     # scope_distance_modifier = 2
 
     async def setup(self):
+        self.historical_logs = int(self.options.get("historical_logs", 7))
+        self.output_dir = self.scan.home / "workflow_logs"
+        self.helpers.mkdir(self.output_dir)
         return await super().setup()
 
     async def filter_event(self, event):
@@ -32,33 +38,15 @@ async def handle_event(self, event):
             self.log.debug(f"Looking up runs for {workflow_name} in {owner}/{repo}")
             for run in await self.get_workflow_runs(owner, repo, workflow_id):
                 run_id = run.get("id")
-                self.log.debug(f"Looking up jobs for {workflow_name}/{run_id} in {owner}/{repo}")
-                for job in await self.get_run_jobs(owner, repo, run_id):
-                    job_id = job.get("id")
-                    commit_id = job.get("head_sha")
-                    steps = job.get("steps", [])
-                    for step in steps:
-                        if step.get("conclusion") == "success":
-                            step_name = step.get("name")
-                            number = step.get("number")
-                            self.log.debug(
-                                f"Requesting {workflow_name}/run {run_id}/job {job_id}/{step_name} log for {owner}/{repo}"
-                            )
-                            # Request log step from the html_url as that bypasses the admin restrictions from using the API
-                            response = await self.helpers.request(
-                                f"https://github.com/{owner}/{repo}/commit/{commit_id}/checks/{job_id}/logs/{number}",
-                                follow_redirects=True,
-                            )
-                            if response:
-                                blob_url = response.headers.get("Location", "")
-                                if blob_url:
-                                    url_event = self.make_event(
-                                        blob_url, "URL_UNVERIFIED", source=event, tags=["httpx-safe"]
-                                    )
-                                    if not url_event:
-                                        continue
-                                    url_event.scope_distance = event.scope_distance
-                                    await self.emit_event(url_event)
+                self.log.debug(f"Downloading logs for {workflow_name}/{run_id} in {owner}/{repo}")
+                log_path = await self.download_run_logs(owner, repo, run_id)
+                if log_path:
+                    self.verbose(f"Downloaded repository workflow logs to {log_path}")
+                    logfile_event = self.make_event(
+                        {"path": str(log_path)}, "FILESYSTEM", tags=["zipfile"], source=event
+                    )
+                    logfile_event.scope_distance = event.scope_distance
+                    await self.emit_event(logfile_event)
 
     async def get_workflows(self, owner, repo):
         workflows = []
@@ -89,8 +77,11 @@ async def get_workflows(self, owner, repo):
 
     async def get_workflow_runs(self, owner, repo, workflow_id):
         runs = []
+        created_date = date.today() - timedelta(days=self.historical_logs)
+        formated_date = created_date.strftime("%Y-%m-%d")
         url = (
-            f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?per_page=100&page=" + "{page}"
+            f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?created=>{formated_date}&per_page=100&page="
+            + "{page}"
         )
         agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
         try:
@@ -116,29 +107,14 @@ async def get_workflow_runs(self, owner, repo, workflow_id):
             agen.aclose()
         return runs
 
-    async def get_run_jobs(self, owner, repo, run_id):
-        jobs = []
-        url = f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/jobs?per_page=100&page=" + "{page}"
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
-        try:
-            async for r in agen:
-                if r is None:
-                    break
-                status_code = getattr(r, "status_code", 0)
-                if status_code == 403:
-                    self.warning("Github is rate-limiting us (HTTP status: 403)")
-                    break
-                if status_code != 200:
-                    break
-                try:
-                    j = r.json()
-                except Exception as e:
-                    self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-                    break
-                if not j:
-                    break
-                for item in j.get("jobs", []):
-                    jobs.append(item)
-        finally:
-            agen.aclose()
-        return jobs
+    async def download_run_logs(self, owner, repo, run_id):
+        file_destination = self.output_dir / f"{owner}_{repo}_run_{run_id}.zip"
+        result = await self.helpers.download(
+            f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs", filename=file_destination
+        )
+        if result:
+            self.info(f"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}")
+            return file_destination
+        else:
+            self.warning(f"Failed to download logs for {owner}/{repo}/{run_id}")
+            return None
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
index ee2f93a49..ed4ce921b 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -415,118 +415,16 @@ async def setup_before_prep(self, module_test):
             },
         )
         module_test.httpx_mock.add_response(
-            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/jobs?per_page=100&page=1",
-            json={
-                "total_count": 8,
-                "jobs": [
-                    {
-                        "id": 24272553740,
-                        "run_id": 8839360698,
-                        "workflow_name": "tests",
-                        "head_branch": "dnsbrute-helperify",
-                        "run_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698",
-                        "run_attempt": 1,
-                        "node_id": "CR_kwDOG_O3ns8AAAAFpsHHDA",
-                        "head_sha": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
-                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/jobs/24272553740",
-                        "html_url": "https://github.com/blacklanternsecurity/bbot/actions/runs/8839360698/job/24272553740",
-                        "status": "completed",
-                        "conclusion": "success",
-                        "created_at": "2024-04-25T21:04:54Z",
-                        "started_at": "2024-04-25T21:05:01Z",
-                        "completed_at": "2024-04-25T21:05:18Z",
-                        "name": "lint",
-                        "steps": [
-                            {
-                                "name": "Set up job",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 1,
-                                "started_at": "2024-04-25T21:05:00.000Z",
-                                "completed_at": "2024-04-25T21:05:02.000Z",
-                            },
-                            {
-                                "name": "Run actions/checkout@v3",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 2,
-                                "started_at": "2024-04-25T21:05:02.000Z",
-                                "completed_at": "2024-04-25T21:05:04.000Z",
-                            },
-                            {
-                                "name": "Run psf/black@stable",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 3,
-                                "started_at": "2024-04-25T21:05:04.000Z",
-                                "completed_at": "2024-04-25T21:05:10.000Z",
-                            },
-                            {
-                                "name": "Install Python 3",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 4,
-                                "started_at": "2024-04-25T21:05:10.000Z",
-                                "completed_at": "2024-04-25T21:05:11.000Z",
-                            },
-                            {
-                                "name": "Install dependencies",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 5,
-                                "started_at": "2024-04-25T21:05:11.000Z",
-                                "completed_at": "2024-04-25T21:05:12.000Z",
-                            },
-                            {
-                                "name": "flake8",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 6,
-                                "started_at": "2024-04-25T21:05:12.000Z",
-                                "completed_at": "2024-04-25T21:05:14.000Z",
-                            },
-                            {
-                                "name": "Post Install Python 3",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 11,
-                                "started_at": "2024-04-25T21:05:15.000Z",
-                                "completed_at": "2024-04-25T21:05:15.000Z",
-                            },
-                            {
-                                "name": "Post Run actions/checkout@v3",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 12,
-                                "started_at": "2024-04-25T21:05:15.000Z",
-                                "completed_at": "2024-04-25T21:05:15.000Z",
-                            },
-                            {
-                                "name": "Complete job",
-                                "status": "completed",
-                                "conclusion": "success",
-                                "number": 13,
-                                "started_at": "2024-04-25T21:05:15.000Z",
-                                "completed_at": "2024-04-25T21:05:15.000Z",
-                            },
-                        ],
-                        "check_run_url": "https://api.github.com/repos/blacklanternsecurity/bbot/check-runs/24272553740",
-                        "labels": ["ubuntu-latest"],
-                        "runner_id": 60,
-                        "runner_name": "GitHub Actions 60",
-                        "runner_group_id": 2,
-                        "runner_group_name": "GitHub Actions",
-                    },
-                ],
-            },
-        )
-        module_test.httpx_mock.add_response(
-            url="https://github.com/blacklanternsecurity/bbot/commit/c5de1360e8e5ccba04b23035f675a529282b7dc2/checks/24272553740/logs/1",
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/logs",
             headers={
                 "location": "https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02"
             },
             status_code=302,
         )
+        module_test.httpx_mock.add_response(
+            url="https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02",
+            content=self.zip_content,
+        )
 
     def check(self, module_test, events):
         assert len(events) == 5
@@ -565,9 +463,9 @@ def check(self, module_test, events):
             [
                 e
                 for e in events
-                if e.type == "URL_UNVERIFIED"
-                and e.data
-                == "https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02"
+                if e.type == "FILESYSTEM"
+                and "blacklanternsecurity/bbot/run_8839360698.zip" in e.data
+                and "zipfile" in e.tags
                 and e.scope_distance == 1
             ]
         ), "Failed to obtain redirect to the blob"

From 8aacd711de1b8fc4330c8c2001ac202ee4dd67e6 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 13:07:04 -0400
Subject: [PATCH 026/220] fix tests

---
 bbot/test/test_step_1/test_manager_deduplication.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index a1d7f8596..3f88576f9 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -75,6 +75,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
         )
 
     dns_mock_chain = {
+        "test.notreal": {"A": ["127.0.0.3"]},
         "default_module.test.notreal": {"A": ["127.0.0.3"]},
         "everything_module.test.notreal": {"A": ["127.0.0.4"]},
         "no_suppress_dupes.test.notreal": {"A": ["127.0.0.5"]},
@@ -116,7 +117,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
-    assert len(all_events) == 26
+    assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
@@ -127,6 +128,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])

From 7078612a3b2a2193fcc3821ba96c7270f58b4f5f Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Mon, 29 Apr 2024 18:14:26 +0100
Subject: [PATCH 027/220] Changed to grab owner and repo from repo_url

---
 bbot/modules/github_workflows.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 08ca162c5..a018adfa1 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -24,14 +24,14 @@ async def setup(self):
 
     async def filter_event(self, event):
         if event.type == "CODE_REPOSITORY":
-            if "git" not in event.tags:
+            if "git" not in event.tags and "github" not in event.data.get("url", ""):
                 return False, "event is not a git repository"
         return True
 
     async def handle_event(self, event):
-        # repo_url = event.data.get("url")
-        owner = "blacklanternsecurity"
-        repo = "bbot"
+        repo_url = event.data.get("url")
+        owner = repo_url.split("/")[-2]
+        repo = repo_url.split("/")[-1]
         for workflow in await self.get_workflows(owner, repo):
             workflow_name = workflow.get("name")
             workflow_id = workflow.get("id")

From cdeb6a14ec917b196e5bacfd72655c81a1881e4d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 13:30:41 -0400
Subject: [PATCH 028/220] fix azure realm tests

---
 bbot/test/test_step_2/module_tests/test_module_azure_realm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bbot/test/test_step_2/module_tests/test_module_azure_realm.py b/bbot/test/test_step_2/module_tests/test_module_azure_realm.py
index 7ab5463c1..8f5cbcedf 100644
--- a/bbot/test/test_step_2/module_tests/test_module_azure_realm.py
+++ b/bbot/test/test_step_2/module_tests/test_module_azure_realm.py
@@ -20,6 +20,7 @@ class TestAzure_Realm(ModuleTestBase):
     }
 
     async def setup_after_prep(self, module_test):
+        await module_test.mock_dns({"evilcorp.com": {"A": ["127.0.0.5"]}})
         module_test.httpx_mock.add_response(
             url=f"https://login.microsoftonline.com/getuserrealm.srf?login=test@evilcorp.com",
             json=self.response_json,

From 5f3948a6e7f8e13505f220ca9579c763d7e832ab Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 16:14:18 -0400
Subject: [PATCH 029/220] fix dnsbrute tests, fix
 https://github.com/blacklanternsecurity/bbot/issues/1315

---
 bbot/modules/dnsbrute.py                      |  6 +-
 bbot/modules/templates/subdomain_enum.py      | 41 +++++--------
 .../module_tests/test_module_dnsbrute.py      | 61 ++++++++++++-------
 3 files changed, 57 insertions(+), 51 deletions(-)

diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py
index 8dc014084..429a55620 100644
--- a/bbot/modules/dnsbrute.py
+++ b/bbot/modules/dnsbrute.py
@@ -16,7 +16,7 @@ class dnsbrute(subdomain_enum):
     }
     deps_common = ["massdns"]
     reject_wildcards = "strict"
-    dedup_strategy = "parent_domain"
+    dedup_strategy = "lowest_parent"
     _qsize = 10000
 
     async def setup(self):
@@ -25,8 +25,8 @@ async def setup(self):
         self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
         return await super().setup()
 
-    async def eligible_for_enumeration(self, event):
-        eligible, reason = await super().eligible_for_enumeration(event)
+    async def filter_event(self, event):
+        eligible, reason = await super().filter_event(event)
         query = self.make_query(event)
 
         # limit brute force depth
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 3c65dfa34..18243c393 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -26,28 +26,9 @@ class subdomain_enum(BaseModule):
 
     # how to deduplicate incoming events
     # options:
-    #   "root_domain": if a dns name has already been tried, don't try any of its children
-    #   "parent_domain": always try a domain unless its direct parent has already been tried
-    dedup_strategy = "root_domain"
-
-    async def setup(self):
-        strict_scope = self.dedup_strategy == "parent_domain"
-        self.processed = self.helpers.make_target(strict_scope=strict_scope)
-        return True
-
-    async def filter_event(self, event):
-        """
-        This filter_event is used across many modules
-        """
-        query = self.make_query(event)
-        # reject if already processed
-        if query in self.processed:
-            return False, "Event was already processed"
-        eligible, reason = await self.eligible_for_enumeration(event)
-        if eligible:
-            self.processed.add(query)
-            return True, reason
-        return False, reason
+    #   "highest_parent": dedupe by highest parent (highest parent of www.api.test.evilcorp.com is evilcorp.com)
+    #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
+    dedup_strategy = "highest_parent"
 
     async def handle_event(self, event):
         query = self.make_query(event)
@@ -68,10 +49,18 @@ async def request_url(self, query):
         return await self.request_with_fail_count(url)
 
     def make_query(self, event):
-        if "target" in event.tags:
-            query = str(event.data)
+        query = event.data
+        parents = list(self.helpers.domain_parents(event.data))
+        if self.dedup_strategy == "highest_parent":
+            parents = list(reversed(parents))
+        elif self.dedup_strategy == "lowest_parent":
+            pass
         else:
-            query = self.helpers.parent_domain(event.data).lower()
+            raise ValueError('self.dedup_strategy attribute must be set to either "highest_parent" or "lowest_parent"')
+        for p in parents:
+            if self.scan.in_scope(p):
+                query = p
+                break
         return ".".join([s for s in query.split(".") if s != "_wildcard"])
 
     def parse_results(self, r, query=None):
@@ -114,7 +103,7 @@ async def _is_wildcard(self, query):
                     return True
         return False
 
-    async def eligible_for_enumeration(self, event):
+    async def filter_event(self, event):
         query = self.make_query(event)
         # check if wildcard
         is_wildcard = await self._is_wildcard(query)
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
index fab736cca..bdbd2f6cb 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
@@ -3,6 +3,7 @@
 
 class TestDnsbrute(ModuleTestBase):
     subdomain_wordlist = tempwordlist(["www", "asdf"])
+    blacklist = ["api.asdf.blacklanternsecurity.com"]
     config_overrides = {"modules": {"dnsbrute": {"wordlist": str(subdomain_wordlist), "max_depth": 3}}}
 
     async def setup_after_prep(self, module_test):
@@ -24,52 +25,68 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
             }
         )
 
+        module = module_test.module
+        scan = module_test.scan
+
+        # test query logic
+        event = scan.make_event("blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "blacklanternsecurity.com"
+        event = scan.make_event("asdf.blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "blacklanternsecurity.com"
+        event = scan.make_event("api.asdf.blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "asdf.blacklanternsecurity.com"
+        event = scan.make_event("test.api.asdf.blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "asdf.blacklanternsecurity.com"
+
+        assert module.dedup_strategy == "lowest_parent"
+        module.dedup_strategy = "highest_parent"
+        event = scan.make_event("blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "blacklanternsecurity.com"
+        event = scan.make_event("asdf.blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "blacklanternsecurity.com"
+        event = scan.make_event("api.asdf.blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "blacklanternsecurity.com"
+        event = scan.make_event("test.api.asdf.blacklanternsecurity.com", "DNS_NAME", dummy=True)
+        assert module.make_query(event) == "blacklanternsecurity.com"
+        module.dedup_strategy = "lowest_parent"
+
         # test recursive brute-force event filtering
-        event = module_test.scan.make_event("evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event = module_test.scan.make_event("blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event)
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
-        assert "evilcorp.com" in module_test.module.processed
-        assert not "com" in module_test.module.processed
-        event = module_test.scan.make_event("evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
-        event.scope_distance = 0
-        result, reason = await module_test.module.filter_event(event)
-        assert result == False
-        assert reason == "Event was already processed"
-        event = module_test.scan.make_event("www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
-        event.scope_distance = 0
-        result, reason = await module_test.module.filter_event(event)
-        assert result == False
-        assert reason == "Event was already processed"
-        event = module_test.scan.make_event("test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event = module_test.scan.make_event(
+            "www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+        )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
-        event = module_test.scan.make_event("test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event)
+        event = module_test.scan.make_event(
+            "test.www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+        )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
-        assert result == False
-        assert reason == "Event was already processed"
+        assert result == True
         event = module_test.scan.make_event(
-            "asdf.test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event
+            "asdf.test.www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
         event = module_test.scan.make_event(
-            "wat.asdf.test.www.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event
+            "wat.asdf.test.www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == False
-        assert reason == f"subdomain depth of *.asdf.test.www.evilcorp.com (4) > max_depth (3)"
+        assert reason == f"subdomain depth of *.asdf.test.www.blacklanternsecurity.com (4) > max_depth (3)"
         event = module_test.scan.make_event(
-            "hmmm.ptr1234.evilcorp.com", "DNS_NAME", source=module_test.scan.root_event
+            "hmmm.ptr1234.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == False
-        assert reason == f'"ptr1234.evilcorp.com" looks like an autogenerated PTR'
+        assert reason == f'"ptr1234.blacklanternsecurity.com" looks like an autogenerated PTR'
 
     def check(self, module_test, events):
         assert len(events) == 3

From c7a78aeed4fbeeec50448234e4b1025f49d5b723 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Mon, 29 Apr 2024 21:14:55 +0100
Subject: [PATCH 030/220] Break out of api_page_iter properly

---
 bbot/modules/github_workflows.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index a018adfa1..a151217e9 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -14,7 +14,7 @@ class github_workflows(github):
         "historical_logs": "Fetch logs that are at most this many days old (default: 7)",
     }
 
-    # scope_distance_modifier = 2
+    scope_distance_modifier = 2
 
     async def setup(self):
         self.historical_logs = int(self.options.get("historical_logs", 7))
@@ -63,13 +63,13 @@ async def get_workflows(self, owner, repo):
                 if status_code != 200:
                     break
                 try:
-                    j = r.json()
+                    j = r.json().get("workflows", [])
                 except Exception as e:
                     self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
                     break
                 if not j:
                     break
-                for item in j.get("workflows", []):
+                for item in j:
                     workflows.append(item)
         finally:
             agen.aclose()
@@ -95,13 +95,13 @@ async def get_workflow_runs(self, owner, repo, workflow_id):
                 if status_code != 200:
                     break
                 try:
-                    j = r.json()
+                    j = r.json().get("workflow_runs", [])
                 except Exception as e:
                     self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
                     break
                 if not j:
                     break
-                for item in j.get("workflow_runs", []):
+                for item in j:
                     runs.append(item)
         finally:
             agen.aclose()

From e66d0d054ba26f17f4b19471440564c3003d64d9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 16:32:10 -0400
Subject: [PATCH 031/220] fix more tests

---
 bbot/test/test_step_2/module_tests/test_module_baddns_zone.py  | 2 +-
 bbot/test/test_step_2/module_tests/test_module_csv.py          | 2 +-
 bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py b/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py
index b3810e75a..2e0328320 100644
--- a/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py
+++ b/bbot/test/test_step_2/module_tests/test_module_baddns_zone.py
@@ -48,7 +48,7 @@ async def setup_after_prep(self, module_test):
 
         await module_test.mock_dns(
             {
-                "bad.dns": {"NSEC": ["asdf.bad.dns"]},
+                "bad.dns": {"A": ["127.0.0.5"], "NSEC": ["asdf.bad.dns"]},
                 "asdf.bad.dns": {"NSEC": ["zzzz.bad.dns"]},
                 "zzzz.bad.dns": {"NSEC": ["xyz.bad.dns"]},
             }
diff --git a/bbot/test/test_step_2/module_tests/test_module_csv.py b/bbot/test/test_step_2/module_tests/test_module_csv.py
index 0d6e326a9..9e906cba2 100644
--- a/bbot/test/test_step_2/module_tests/test_module_csv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_csv.py
@@ -3,7 +3,7 @@
 
 class TestCSV(ModuleTestBase):
     async def setup_after_prep(self, module_test):
-        await module_test.mock_dns({})
+        await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.5"]}})
 
     def check(self, module_test, events):
         csv_file = module_test.scan.home / "output.csv"
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
index 5850fbd49..c44e3d90a 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
@@ -8,6 +8,7 @@ class TestDNSCommonSRV(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         await module_test.mock_dns(
             {
+                "blacklanternsecurity.notreal": {"A": ["127.0.0.5"]},
                 "_ldap._tcp.gc._msdcs.blacklanternsecurity.notreal": {
                     "SRV": ["0 100 3268 asdf.blacklanternsecurity.notreal"]
                 },

From 906b4440ef8bea4efd9700fe7c3e7bfa5040406c Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 29 Apr 2024 16:42:27 -0400
Subject: [PATCH 032/220] fix more tests

---
 bbot/test/test_step_2/module_tests/test_module_csv.py    | 2 +-
 bbot/test/test_step_2/module_tests/test_module_oauth.py  | 1 +
 .../test/test_step_2/module_tests/test_module_postman.py | 4 +++-
 .../test_step_2/module_tests/test_module_sitedossier.py  | 9 +++++++++
 4 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_csv.py b/bbot/test/test_step_2/module_tests/test_module_csv.py
index 9e906cba2..6dc3f7cf1 100644
--- a/bbot/test/test_step_2/module_tests/test_module_csv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_csv.py
@@ -8,4 +8,4 @@ async def setup_after_prep(self, module_test):
     def check(self, module_test, events):
         csv_file = module_test.scan.home / "output.csv"
         with open(csv_file) as f:
-            assert "DNS_NAME,blacklanternsecurity.com,,TARGET" in f.read()
+            assert "blacklanternsecurity.com,127.0.0.5,TARGET" in f.read()
diff --git a/bbot/test/test_step_2/module_tests/test_module_oauth.py b/bbot/test/test_step_2/module_tests/test_module_oauth.py
index eac2db2ac..562f90c84 100644
--- a/bbot/test/test_step_2/module_tests/test_module_oauth.py
+++ b/bbot/test/test_step_2/module_tests/test_module_oauth.py
@@ -165,6 +165,7 @@ class TestOAUTH(ModuleTestBase):
     }
 
     async def setup_after_prep(self, module_test):
+        await module_test.mock_dns({"evilcorp.com": {"A": ["127.0.0.1"]}})
         module_test.httpx_mock.add_response(
             url=f"https://login.microsoftonline.com/getuserrealm.srf?login=test@evilcorp.com",
             json=Azure_Realm.response_json,
diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py
index 8e9c0f3bf..a15ac7ede 100644
--- a/bbot/test/test_step_2/module_tests/test_module_postman.py
+++ b/bbot/test/test_step_2/module_tests/test_module_postman.py
@@ -235,7 +235,9 @@ async def new_emit_event(event_data, event_type, **kwargs):
             await old_emit_event(event_data, event_type, **kwargs)
 
         module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event)
-        await module_test.mock_dns({"asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}})
+        await module_test.mock_dns(
+            {"blacklanternsecurity.com": {"A": ["127.0.0.1"]}, "asdf.blacklanternsecurity.com": {"A": ["127.0.0.1"]}}
+        )
 
         request_args = dict(uri="/_api/request/28129865-987c8ac8-bfa9-4bab-ade9-88ccf0597862")
         respond_args = dict(response_data="https://asdf.blacklanternsecurity.com")
diff --git a/bbot/test/test_step_2/module_tests/test_module_sitedossier.py b/bbot/test/test_step_2/module_tests/test_module_sitedossier.py
index 2156a5ae7..a5b57b800 100644
--- a/bbot/test/test_step_2/module_tests/test_module_sitedossier.py
+++ b/bbot/test/test_step_2/module_tests/test_module_sitedossier.py
@@ -126,6 +126,15 @@ class TestSitedossier(ModuleTestBase):
     targets = ["evilcorp.com"]
 
     async def setup_after_prep(self, module_test):
+        await module_test.mock_dns(
+            {
+                "evilcorp.com": {"A": ["127.0.0.1"]},
+                "asdf.evilcorp.com": {"A": ["127.0.0.1"]},
+                "zzzz.evilcorp.com": {"A": ["127.0.0.1"]},
+                "xxxx.evilcorp.com": {"A": ["127.0.0.1"]},
+                "ffff.evilcorp.com": {"A": ["127.0.0.1"]},
+            }
+        )
         module_test.httpx_mock.add_response(
             url=f"http://www.sitedossier.com/parentdomain/evilcorp.com",
             text=page1,

From 46f1ddd3ee7937d83bb4d5dac8c434bc6afb6a53 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Tue, 30 Apr 2024 08:55:29 +0100
Subject: [PATCH 033/220] Add workflow logs into their own folder

---
 bbot/modules/github_workflows.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index a151217e9..edfd4b403 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -108,9 +108,12 @@ async def get_workflow_runs(self, owner, repo, workflow_id):
         return runs
 
     async def download_run_logs(self, owner, repo, run_id):
-        file_destination = self.output_dir / f"{owner}_{repo}_run_{run_id}.zip"
+        folder = self.output_dir / owner / repo
+        self.helpers.mkdir(folder)
+        filename = f"run_{run_id}.zip"
+        file_destination = folder / filename
         result = await self.helpers.download(
-            f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs", filename=file_destination
+            f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs", filename=file_destination, headers=self.headers
         )
         if result:
             self.info(f"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}")

From 40f99ddeaac075b698e71bc95810a26a422d871d Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Tue, 30 Apr 2024 10:19:36 +0100
Subject: [PATCH 034/220] Corrected tests and formatted

---
 bbot/modules/github_workflows.py              |  4 +-
 .../test_module_github_workflows.py           | 50 ++++++++++++-------
 2 files changed, 36 insertions(+), 18 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index edfd4b403..170db1190 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -113,7 +113,9 @@ async def download_run_logs(self, owner, repo, run_id):
         filename = f"run_{run_id}.zip"
         file_destination = folder / filename
         result = await self.helpers.download(
-            f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs", filename=file_destination, headers=self.headers
+            f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs",
+            filename=file_destination,
+            headers=self.headers,
         )
         if result:
             self.info(f"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}")
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
index ed4ce921b..18997b910 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -1,3 +1,7 @@
+import io
+import zipfile
+from pathlib import Path
+
 from .base import ModuleTestBase
 
 
@@ -5,6 +9,13 @@ class TestGithub_Workflows(ModuleTestBase):
     config_overrides = {"modules": {"github_org": {"api_key": "asdf"}}}
     modules_overrides = ["github_workflows", "github_org", "speculate"]
 
+    data = io.BytesIO()
+    with zipfile.ZipFile(data, mode="w", compression=zipfile.ZIP_DEFLATED) as zipfile:
+        zipfile.writestr("test.txt", "This is some test data")
+    data.seek(0)
+
+    zip_content = data.getvalue()
+
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(url="https://api.github.com/zen")
         module_test.httpx_mock.add_response(
@@ -71,7 +82,7 @@ async def setup_before_prep(self, module_test):
                         "type": "Organization",
                         "site_admin": False,
                     },
-                    "html_url": "https://github.com/blacklanternsecurity/test_keys",
+                    "html_url": "https://github.com/blacklanternsecurity/bbot",
                     "description": None,
                     "fork": False,
                     "url": "https://api.github.com/repos/blacklanternsecurity/test_keys",
@@ -116,8 +127,8 @@ async def setup_before_prep(self, module_test):
                     "pushed_at": "2023-10-19T02:56:46Z",
                     "git_url": "git://github.com/blacklanternsecurity/test_keys.git",
                     "ssh_url": "git@github.com:blacklanternsecurity/test_keys.git",
-                    "clone_url": "https://github.com/blacklanternsecurity/test_keys.git",
-                    "svn_url": "https://github.com/blacklanternsecurity/test_keys",
+                    "clone_url": "https://github.com/blacklanternsecurity/bbot.git",
+                    "svn_url": "https://github.com/blacklanternsecurity/bbot",
                     "homepage": None,
                     "size": 2,
                     "stargazers_count": 2,
@@ -169,7 +180,7 @@ async def setup_before_prep(self, module_test):
             },
         )
         module_test.httpx_mock.add_response(
-            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?per_page=100&page=1",
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?created=>2024-04-23&per_page=100&page=1",
             json={
                 "total_count": 2993,
                 "workflow_runs": [
@@ -415,7 +426,7 @@ async def setup_before_prep(self, module_test):
             },
         )
         module_test.httpx_mock.add_response(
-            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/logs",
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/logs",
             headers={
                 "location": "https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02"
             },
@@ -427,7 +438,7 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-        assert len(events) == 5
+        assert len(events) == 6
         assert 1 == len(
             [
                 e
@@ -455,17 +466,22 @@ def check(self, module_test, events):
                 for e in events
                 if e.type == "CODE_REPOSITORY"
                 and "git" in e.tags
-                and e.data["url"] == "https://github.com/blacklanternsecurity/test_keys"
+                and e.data["url"] == "https://github.com/blacklanternsecurity/bbot"
                 and e.scope_distance == 1
             ]
         ), "Failed to find blacklanternsecurity github repo"
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.type == "FILESYSTEM"
-                and "blacklanternsecurity/bbot/run_8839360698.zip" in e.data
-                and "zipfile" in e.tags
-                and e.scope_distance == 1
-            ]
-        ), "Failed to obtain redirect to the blob"
+        filesystem_events = [
+            e
+            for e in events
+            if e.type == "FILESYSTEM"
+            and "workflow_logs/blacklanternsecurity/bbot/run_8839360698.zip" in e.data["path"]
+            and "zipfile" in e.tags
+            and e.scope_distance == 1
+        ]
+        assert 1 == len(filesystem_events), "Failed to download workflow logs"
+        filesystem_event = filesystem_events[0]
+        file = Path(filesystem_event.data["path"])
+        assert file.is_file(), "Destination zip does not exist"
+        with zipfile.ZipFile(file, "r") as zip_ref:
+            assert "test.txt" in zip_ref.namelist(), "test.txt not in zip"
+            assert zip_ref.read("test.txt") == b"This is some test data", "test.txt contents incorrect"

From ca56eb33aa3cef418640e858e12ea1b6ee3685f5 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Tue, 30 Apr 2024 10:50:54 +0100
Subject: [PATCH 035/220] Match URL using regex

---
 .../module_tests/test_module_github_workflows.py            | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
index 18997b910..8961b63c0 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -1,4 +1,5 @@
 import io
+import re
 import zipfile
 from pathlib import Path
 
@@ -13,7 +14,6 @@ class TestGithub_Workflows(ModuleTestBase):
     with zipfile.ZipFile(data, mode="w", compression=zipfile.ZIP_DEFLATED) as zipfile:
         zipfile.writestr("test.txt", "This is some test data")
     data.seek(0)
-
     zip_content = data.getvalue()
 
     async def setup_before_prep(self, module_test):
@@ -180,7 +180,9 @@ async def setup_before_prep(self, module_test):
             },
         )
         module_test.httpx_mock.add_response(
-            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?created=>2024-04-23&per_page=100&page=1",
+            url=re.compile(
+                r"https://api\.github\.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs\?created=.*&per_page=100&page=1"
+            ),
             json={
                 "total_count": 2993,
                 "workflow_runs": [

From 98c3281f3a5e642e397aa65a5f83e8baf4ac082e Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Tue, 30 Apr 2024 11:47:30 +0100
Subject: [PATCH 036/220] Change the workflow logs obtained to be configurable

---
 bbot/modules/github_workflows.py              | 56 ++++++++-----------
 .../test_module_github_workflows.py           |  7 +--
 2 files changed, 26 insertions(+), 37 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 170db1190..dc0f63ed7 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -1,5 +1,3 @@
-from datetime import date, timedelta
-
 from bbot.modules.templates.github import github
 
 
@@ -8,16 +6,19 @@ class github_workflows(github):
     produced_events = ["FILESYSTEM"]
     flags = ["passive", "safe"]
     meta = {"description": "Download a github repositories workflow logs"}
-    options = {"api_key": "", "historical_logs": 7}
+    options = {"api_key": "", "num_logs": 1}
     options_desc = {
         "api_key": "Github token",
-        "historical_logs": "Fetch logs that are at most this many days old (default: 7)",
+        "num_logs": "For each workflow fetch the last N successful runs logs (max 100)",
     }
 
     scope_distance_modifier = 2
 
     async def setup(self):
-        self.historical_logs = int(self.options.get("historical_logs", 7))
+        self.num_logs = int(self.options.get("num_logs", 1))
+        if self.num_logs > 100:
+            self.log.error("num_logs option is capped at 100")
+            return False
         self.output_dir = self.scan.home / "workflow_logs"
         self.helpers.mkdir(self.output_dir)
         return await super().setup()
@@ -77,34 +78,25 @@ async def get_workflows(self, owner, repo):
 
     async def get_workflow_runs(self, owner, repo, workflow_id):
         runs = []
-        created_date = date.today() - timedelta(days=self.historical_logs)
-        formated_date = created_date.strftime("%Y-%m-%d")
-        url = (
-            f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?created=>{formated_date}&per_page=100&page="
-            + "{page}"
-        )
-        agen = self.helpers.api_page_iter(url, headers=self.headers, json=False)
+        url = f"{self.base_url}/repos/{owner}/{repo}/actions/workflows/{workflow_id}/runs?status=success&per_page={self.num_logs}"
+        r = await self.helpers.request(url, headers=self.headers)
+        if r is None:
+            return runs
+        status_code = getattr(r, "status_code", 0)
+        if status_code == 403:
+            self.warning("Github is rate-limiting us (HTTP status: 403)")
+            return runs
+        if status_code != 200:
+            return runs
         try:
-            async for r in agen:
-                if r is None:
-                    break
-                status_code = getattr(r, "status_code", 0)
-                if status_code == 403:
-                    self.warning("Github is rate-limiting us (HTTP status: 403)")
-                    break
-                if status_code != 200:
-                    break
-                try:
-                    j = r.json().get("workflow_runs", [])
-                except Exception as e:
-                    self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
-                    break
-                if not j:
-                    break
-                for item in j:
-                    runs.append(item)
-        finally:
-            agen.aclose()
+            j = r.json().get("workflow_runs", [])
+        except Exception as e:
+            self.warning(f"Failed to decode JSON for {r.url} (HTTP status: {status_code}): {e}")
+            return runs
+        if not j:
+            return runs
+        for item in j:
+            runs.append(item)
         return runs
 
     async def download_run_logs(self, owner, repo, run_id):
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
index 8961b63c0..76bd55814 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -1,5 +1,4 @@
 import io
-import re
 import zipfile
 from pathlib import Path
 
@@ -180,9 +179,7 @@ async def setup_before_prep(self, module_test):
             },
         )
         module_test.httpx_mock.add_response(
-            url=re.compile(
-                r"https://api\.github\.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs\?created=.*&per_page=100&page=1"
-            ),
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?status=success&per_page=1",
             json={
                 "total_count": 2993,
                 "workflow_runs": [
@@ -197,7 +194,7 @@ async def setup_before_prep(self, module_test):
                         "run_number": 4520,
                         "event": "pull_request",
                         "status": "completed",
-                        "conclusion": "failure",
+                        "conclusion": "success",
                         "workflow_id": 22452226,
                         "check_suite_id": 23162098295,
                         "check_suite_node_id": "CS_kwDOG_O3ns8AAAAFZJGSdw",

From 795a4c9faeb407fe9ed006cd3c748af66e4020fb Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Tue, 30 Apr 2024 12:09:32 +0100
Subject: [PATCH 037/220] Change config to download a set number of logs

---
 bbot/modules/github_workflows.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index dc0f63ed7..b4762ccbd 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -15,7 +15,7 @@ class github_workflows(github):
     scope_distance_modifier = 2
 
     async def setup(self):
-        self.num_logs = int(self.options.get("num_logs", 1))
+        self.num_logs = int(self.config.get("num_logs", 1))
         if self.num_logs > 100:
             self.log.error("num_logs option is capped at 100")
             return False

From 0fb1c30bab7ed30c8322701df2f9b678d3ad101e Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Tue, 30 Apr 2024 12:25:28 +0100
Subject: [PATCH 038/220] Changed error message

---
 bbot/modules/github_workflows.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index b4762ccbd..35609232c 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -113,5 +113,5 @@ async def download_run_logs(self, owner, repo, run_id):
             self.info(f"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}")
             return file_destination
         else:
-            self.warning(f"Failed to download logs for {owner}/{repo}/{run_id}")
+            self.warning(f"The logs for {owner}/{repo}/{run_id} have expired and are no longer available.")
             return None

From c36bcae8cdaa98df658ec212c77abc407798126a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 30 Apr 2024 17:11:46 -0400
Subject: [PATCH 039/220] fix event type omissions, make exceptions for
 explicit targets

---
 bbot/core/event/base.py                       | 65 ++++++-------------
 bbot/modules/base.py                          |  3 +-
 bbot/modules/bypass403.py                     |  6 +-
 bbot/modules/deadly/dastardly.py              |  2 +-
 bbot/modules/deadly/ffuf.py                   |  2 +-
 bbot/modules/deadly/vhost.py                  | 12 ++--
 bbot/modules/ffuf_shortnames.py               | 10 +--
 bbot/modules/generic_ssrf.py                  |  4 +-
 bbot/modules/gitlab.py                        |  2 +-
 bbot/modules/httpx.py                         |  2 +-
 bbot/modules/internal/excavate.py             |  6 +-
 bbot/modules/ntlm.py                          |  2 +-
 bbot/modules/output/base.py                   | 24 ++++---
 bbot/modules/output/web_report.py             |  4 +-
 bbot/modules/robots.py                        |  2 +-
 bbot/modules/secretsdb.py                     |  2 +-
 bbot/modules/url_manipulation.py              |  6 +-
 bbot/modules/wafw00f.py                       |  2 +-
 bbot/scanner/manager.py                       | 21 +++++-
 bbot/scanner/preset/args.py                   |  2 +-
 bbot/scanner/scanner.py                       |  7 ++
 bbot/test/test_step_1/test_events.py          |  2 +-
 .../module_tests/test_module_dastardly.py     |  2 +-
 23 files changed, 97 insertions(+), 93 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index d7eabd6db..abacbf132 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -15,7 +15,6 @@
 from bbot.errors import *
 from bbot.core.helpers import (
     extract_words,
-    get_file_extension,
     is_domain,
     is_subdomain,
     is_ip,
@@ -99,8 +98,6 @@ class BaseEvent:
     _quick_emit = False
     # Whether this event has been retroactively marked as part of an important discovery chain
     _graph_important = False
-    # Exclude from output modules
-    _omit = False
     # Disables certain data validations
     _dummy = False
     # Data validation, if data is a dictionary
@@ -149,6 +146,7 @@ def __init__(
         self._hash = None
         self.__host = None
         self._port = None
+        self._omit = False
         self.__words = None
         self._priority = None
         self._host_original = None
@@ -185,9 +183,6 @@ def __init__(
         if self.scan:
             self.scans = list(set([self.scan.id] + self.scans))
 
-        # check type blacklist
-        self._check_omit()
-
         self._scope_distance = -1
 
         try:
@@ -297,12 +292,12 @@ def host_original(self):
     @property
     def port(self):
         self.host
-        if getattr(self, "parsed", None):
-            if self.parsed.port is not None:
-                return self.parsed.port
-            elif self.parsed.scheme == "https":
+        if getattr(self, "parsed_url", None):
+            if self.parsed_url.port is not None:
+                return self.parsed_url.port
+            elif self.parsed_url.scheme == "https":
                 return 443
-            elif self.parsed.scheme == "http":
+            elif self.parsed_url.scheme == "http":
                 return 80
         return self._port
 
@@ -453,9 +448,9 @@ def source_id(self):
 
     def get_source(self):
         """
-        Takes into account events with the _omit flag
+        Takes into account events with the omit flag
         """
-        if getattr(self.source, "_omit", False):
+        if getattr(self.source, "omit", False):
             return self.source.get_source()
         return self.source
 
@@ -669,7 +664,7 @@ def module_sequence(self):
             str: The module sequence in human-friendly format.
         """
         module_name = getattr(self.module, "name", "")
-        if getattr(self.source, "_omit", False):
+        if getattr(self.source, "omit", False):
             module_name = f"{self.source.module_sequence}->{module_name}"
         return module_name
 
@@ -704,13 +699,6 @@ def type(self, val):
         self._type = val
         self._hash = None
         self._id = None
-        self._check_omit()
-
-    def _check_omit(self):
-        if self.scan is not None:
-            omit_event_types = self.scan.config.get("omit_event_types", [])
-            if omit_event_types and self.type in omit_event_types:
-                self._omit = True
 
     def __iter__(self):
         """
@@ -770,7 +758,7 @@ class DictEvent(BaseEvent):
     def sanitize_data(self, data):
         url = data.get("url", "")
         if url:
-            self.parsed = validators.validate_url_parsed(url)
+            self.parsed_url = validators.validate_url_parsed(url)
         return data
 
     def _data_load(self, data):
@@ -784,7 +772,7 @@ def _host(self):
         if isinstance(self.data, dict) and "host" in self.data:
             return make_ip_type(self.data["host"])
         else:
-            parsed = getattr(self, "parsed")
+            parsed = getattr(self, "parsed_url", None)
             if parsed is not None:
                 return make_ip_type(parsed.hostname)
 
@@ -903,44 +891,29 @@ def __init__(self, *args, **kwargs):
         self.num_redirects = getattr(self.source, "num_redirects", 0)
 
     def sanitize_data(self, data):
-        self.parsed = validators.validate_url_parsed(data)
+        self.parsed_url = validators.validate_url_parsed(data)
 
         # tag as dir or endpoint
-        if str(self.parsed.path).endswith("/"):
+        if str(self.parsed_url.path).endswith("/"):
             self.add_tag("dir")
         else:
             self.add_tag("endpoint")
 
-        parsed_path_lower = str(self.parsed.path).lower()
-
-        scan = getattr(self, "scan", None)
-        url_extension_blacklist = getattr(scan, "url_extension_blacklist", [])
-        url_extension_httpx_only = getattr(scan, "url_extension_httpx_only", [])
-
-        extension = get_file_extension(parsed_path_lower)
-        if extension:
-            self.add_tag(f"extension-{extension}")
-            if extension in url_extension_blacklist:
-                self.add_tag("blacklisted")
-            if extension in url_extension_httpx_only:
-                self.add_tag("httpx-only")
-                self._omit = True
-
-        data = self.parsed.geturl()
+        data = self.parsed_url.geturl()
         return data
 
     def with_port(self):
         netloc_with_port = make_netloc(self.host, self.port)
-        return self.parsed._replace(netloc=netloc_with_port)
+        return self.parsed_url._replace(netloc=netloc_with_port)
 
     def _words(self):
-        first_elem = self.parsed.path.lstrip("/").split("/")[0]
+        first_elem = self.parsed_url.path.lstrip("/").split("/")[0]
         if not "." in first_elem:
             return extract_words(first_elem)
         return set()
 
     def _host(self):
-        return make_ip_type(self.parsed.hostname)
+        return make_ip_type(self.parsed_url.hostname)
 
     def _data_id(self):
         # consider spider-danger tag when deduping
@@ -1020,8 +993,8 @@ def __init__(self, *args, **kwargs):
 
     def sanitize_data(self, data):
         url = data.get("url", "")
-        self.parsed = validators.validate_url_parsed(url)
-        data["url"] = self.parsed.geturl()
+        self.parsed_url = validators.validate_url_parsed(url)
+        data["url"] = self.parsed_url.geturl()
 
         header_dict = {}
         for i in data.get("raw_header", "").splitlines():
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index d1349818c..92e341e18 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -684,6 +684,7 @@ def _event_precheck(self, event):
         if self.target_only:
             if "target" not in event.tags:
                 return False, "it did not meet target_only filter criteria"
+
         # exclude certain URLs (e.g. javascript):
         # TODO: revisit this after httpx rework
         if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
@@ -952,7 +953,7 @@ def get_per_hostport_hash(self, event):
             >>> event = self.make_event("https://example.com:8443")
             >>> self.get_per_hostport_hash(event)
         """
-        parsed = getattr(event, "parsed", None)
+        parsed = getattr(event, "parsed_url", None)
         if parsed is None:
             to_hash = self.helpers.make_netloc(event.host, event.port)
         else:
diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index 0ce3df899..95f3ef551 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -164,10 +164,10 @@ async def filter_event(self, event):
 
     def format_signature(self, sig, event):
         if sig[3] == True:
-            cleaned_path = event.parsed.path.strip("/")
+            cleaned_path = event.parsed_url.path.strip("/")
         else:
-            cleaned_path = event.parsed.path.lstrip("/")
-        kwargs = {"scheme": event.parsed.scheme, "netloc": event.parsed.netloc, "path": cleaned_path}
+            cleaned_path = event.parsed_url.path.lstrip("/")
+        kwargs = {"scheme": event.parsed_url.scheme, "netloc": event.parsed_url.netloc, "path": cleaned_path}
         formatted_url = sig[1].format(**kwargs)
         if sig[2] != None:
             formatted_headers = {k: v.format(**kwargs) for k, v in sig[2].items()}
diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py
index 837b4a2c2..742fca77f 100644
--- a/bbot/modules/deadly/dastardly.py
+++ b/bbot/modules/deadly/dastardly.py
@@ -27,7 +27,7 @@ async def filter_event(self, event):
         return True
 
     async def handle_event(self, event):
-        host = event.parsed._replace(path="/").geturl()
+        host = event.parsed_url._replace(path="/").geturl()
         self.verbose(f"Running Dastardly scan against {host}")
         command, output_file = self.construct_command(host)
         finished_proc = await self.run_process(command, sudo=True)
diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py
index a56c73506..94e42d6be 100644
--- a/bbot/modules/deadly/ffuf.py
+++ b/bbot/modules/deadly/ffuf.py
@@ -57,7 +57,7 @@ async def handle_event(self, event):
             return
 
         # only FFUF against a directory
-        if "." in event.parsed.path.split("/")[-1]:
+        if "." in event.parsed_url.path.split("/")[-1]:
             self.debug("Aborting FFUF as period was detected in right-most path segment (likely a file)")
             return
         else:
diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py
index cf7be1f67..101f41536 100644
--- a/bbot/modules/deadly/vhost.py
+++ b/bbot/modules/deadly/vhost.py
@@ -33,7 +33,7 @@ async def setup(self):
 
     async def handle_event(self, event):
         if not self.helpers.is_ip(event.host) or self.config.get("force_basehost"):
-            host = f"{event.parsed.scheme}://{event.parsed.netloc}"
+            host = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}"
             if host in self.scanned_hosts.keys():
                 return
             else:
@@ -44,7 +44,7 @@ async def handle_event(self, event):
             if self.config.get("force_basehost"):
                 basehost = self.config.get("force_basehost")
             else:
-                basehost = self.helpers.parent_domain(event.parsed.netloc)
+                basehost = self.helpers.parent_domain(event.parsed_url.netloc)
 
             self.debug(f"Using basehost: {basehost}")
             async for vhost in self.ffuf_vhost(host, f".{basehost}", event):
@@ -55,7 +55,7 @@ async def handle_event(self, event):
             # check existing host for mutations
             self.verbose("Checking for vhost mutations on main host")
             async for vhost in self.ffuf_vhost(
-                host, f".{basehost}", event, wordlist=self.mutations_check(event.parsed.netloc.split(".")[0])
+                host, f".{basehost}", event, wordlist=self.mutations_check(event.parsed_url.netloc.split(".")[0])
             ):
                 pass
 
@@ -81,7 +81,7 @@ async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=F
         ):
             found_vhost_b64 = r["input"]["FUZZ"]
             vhost_dict = {"host": str(event.host), "url": host, "vhost": base64.b64decode(found_vhost_b64).decode()}
-            if f"{vhost_dict['vhost']}{basehost}" != event.parsed.netloc:
+            if f"{vhost_dict['vhost']}{basehost}" != event.parsed_url.netloc:
                 await self.emit_event(vhost_dict, "VHOST", source=event)
                 if skip_dns_host == False:
                     await self.emit_event(f"{vhost_dict['vhost']}{basehost}", "DNS_NAME", source=event, tags=["vhost"])
@@ -102,13 +102,13 @@ async def finish(self):
 
         for host, event in self.scanned_hosts.items():
             if host not in self.wordcloud_tried_hosts:
-                event.parsed = urlparse(host)
+                event.parsed_url = urlparse(host)
 
                 self.verbose("Checking main host with wordcloud")
                 if self.config.get("force_basehost"):
                     basehost = self.config.get("force_basehost")
                 else:
-                    basehost = self.helpers.parent_domain(event.parsed.netloc)
+                    basehost = self.helpers.parent_domain(event.parsed_url.netloc)
 
                 async for vhost in self.ffuf_vhost(host, f".{basehost}", event, wordlist=tempfile):
                     pass
diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py
index cfc58cba4..49cb8a779 100644
--- a/bbot/modules/ffuf_shortnames.py
+++ b/bbot/modules/ffuf_shortnames.py
@@ -92,7 +92,7 @@ async def setup(self):
 
     def build_extension_list(self, event):
         used_extensions = []
-        extension_hint = event.parsed.path.rsplit(".", 1)[1].lower().strip()
+        extension_hint = event.parsed_url.path.rsplit(".", 1)[1].lower().strip()
         if len(extension_hint) == 3:
             with open(self.wordlist_extensions) as f:
                 for l in f:
@@ -117,9 +117,9 @@ async def filter_event(self, event):
         return True
 
     async def handle_event(self, event):
-        filename_hint = re.sub(r"~\d", "", event.parsed.path.rsplit(".", 1)[0].split("/")[-1]).lower()
+        filename_hint = re.sub(r"~\d", "", event.parsed_url.path.rsplit(".", 1)[0].split("/")[-1]).lower()
 
-        host = f"{event.source.parsed.scheme}://{event.source.parsed.netloc}/"
+        host = f"{event.source.parsed_url.scheme}://{event.source.parsed_url.netloc}/"
         if host not in self.per_host_collection.keys():
             self.per_host_collection[host] = [(filename_hint, event.source.data)]
 
@@ -128,8 +128,8 @@ async def handle_event(self, event):
 
         self.shortname_to_event[filename_hint] = event
 
-        root_stub = "/".join(event.parsed.path.split("/")[:-1])
-        root_url = f"{event.parsed.scheme}://{event.parsed.netloc}{root_stub}/"
+        root_stub = "/".join(event.parsed_url.path.split("/")[:-1])
+        root_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{root_stub}/"
 
         if "shortname-file" in event.tags:
             used_extensions = self.build_extension_list(event)
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index 42efa5050..efad29f76 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -44,7 +44,7 @@ def __init__(self, parent_module):
         self.test_paths = self.create_paths()
 
     def set_base_url(self, event):
-        return f"{event.parsed.scheme}://{event.parsed.netloc}"
+        return f"{event.parsed_url.scheme}://{event.parsed_url.netloc}"
 
     def create_paths(self):
         return self.paths
@@ -140,7 +140,7 @@ async def test(self, event):
 <!ENTITY % {rand_entity} SYSTEM "http://{subdomain_tag}.{self.parent_module.interactsh_domain}" >
 ]>
 <foo>&{rand_entity};</foo>"""
-        test_url = f"{event.parsed.scheme}://{event.parsed.netloc}/"
+        test_url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
         r = await self.parent_module.helpers.curl(
             url=test_url, method="POST", raw_body=post_body, headers={"Content-type": "application/xml"}
         )
diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py
index 6f4892580..3675adfd3 100644
--- a/bbot/modules/gitlab.py
+++ b/bbot/modules/gitlab.py
@@ -47,7 +47,7 @@ async def handle_http_response(self, event):
         # HTTP_RESPONSE --> FINDING
         headers = event.data.get("header", {})
         if "x_gitlab_meta" in headers:
-            url = event.parsed._replace(path="/").geturl()
+            url = event.parsed_url._replace(path="/").geturl()
             await self.emit_event(
                 {"host": str(event.host), "technology": "GitLab", "url": url}, "TECHNOLOGY", source=event
             )
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index 0f74fbcfc..c5e1a42f3 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -84,7 +84,7 @@ async def handle_batch(self, *events):
             if e.type.startswith("URL"):
                 # we NEED the port, otherwise httpx will try HTTPS even for HTTP URLs
                 url = e.with_port().geturl()
-                if e.parsed.path == "/":
+                if e.parsed_url.path == "/":
                     url_hash = hash((e.host, e.port))
             else:
                 url = str(e.data)
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 6b819c0d2..134124333 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -122,7 +122,7 @@ async def search(self, content, event, **kwargs):
                     urls_found += 1
 
     async def _search(self, content, event, **kwargs):
-        parsed = getattr(event, "parsed", None)
+        parsed = getattr(event, "parsed_url", None)
         for name, regex in self.compiled_regexes.items():
             # yield to event loop
             await self.excavate.helpers.sleep(0)
@@ -142,7 +142,7 @@ async def _search(self, content, event, **kwargs):
                             continue
 
                     if not self.compiled_regexes["fullurl"].match(path):
-                        source_url = event.parsed.geturl()
+                        source_url = event.parsed_url.geturl()
                         result = urljoin(source_url, path)
                         # this is necessary to weed out mailto: and such
                         if not self.compiled_regexes["fullurl"].match(result):
@@ -167,7 +167,7 @@ async def report(self, result, name, event, **kwargs):
             # these findings are pretty mundane so don't bother with them if they aren't in scope
             abort_if = lambda e: e.scope_distance > 0
             event_data = {"host": str(host), "description": f"Non-HTTP URI: {result}"}
-            parsed_url = getattr(event, "parsed", None)
+            parsed_url = getattr(event, "parsed_url", None)
             if parsed_url:
                 event_data["url"] = parsed_url.geturl()
             await self.excavate.emit_event(
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index 93f622566..b08fe4784 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -122,7 +122,7 @@ async def handle_url(self, event):
             }
         if self.try_all:
             for endpoint in ntlm_discovery_endpoints:
-                urls.add(f"{event.parsed.scheme}://{event.parsed.netloc}/{endpoint}")
+                urls.add(f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{endpoint}")
 
         tasks = []
         for url in urls:
diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index c9a41eebd..4a00af74d 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -29,23 +29,27 @@ def _event_precheck(self, event):
         if self.target_only:
             if "target" not in event.tags:
                 return False, "it did not meet target_only filter criteria"
-        # exclude certain URLs (e.g. javascript):
-        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
-            return False, "its extension was listed in url_extension_httpx_only"
 
         ### begin output-module specific ###
 
-        # events from omit_event_types, e.g. HTTP_RESPONSE, DNS_NAME_UNRESOLVED, etc.
-        # if the output module specifically requests a certain event type, we let it through anyway
-        # an exception is also made for targets
-        if event._omit:
-            if not "target" in event.tags and not event.type in self.get_watched_events():
-                return False, "_omit is True"
-
         # force-output certain events to the graph
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
+        # exclude certain URLs (e.g. javascript):
+        # TODO: revisit this after httpx rework
+        if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
+            return False, (f"Omitting {event} from output because it's marked as httpx-only")
+
+        # omit certain event types
+        if event.type in self.scan.omitted_event_types:
+            if "target" in event.tags:
+                self.debug(f"Allowing omitted event: {event} because it's a target")
+            elif event.type in self.get_watched_events():
+                self.debug(f"Allowing omitted event: {event} because its type is explicitly in watched_events")
+            else:
+                return False, f"its type is omitted in the config"
+
         # internal events like those from speculate, ipneighbor
         # or events that are over our report distance
         if event._internal:
diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py
index 793f26c32..246704cc5 100644
--- a/bbot/modules/output/web_report.py
+++ b/bbot/modules/output/web_report.py
@@ -34,7 +34,7 @@ async def setup(self):
 
     async def handle_event(self, event):
         if event.type == "URL":
-            parsed = event.parsed
+            parsed = event.parsed_url
             host = f"{parsed.scheme}://{parsed.netloc}/"
             if host not in self.web_assets.keys():
                 self.web_assets[host] = {"URL": []}
@@ -60,7 +60,7 @@ async def handle_event(self, event):
             parsed = None
             while 1:
                 if current_parent.type == "URL":
-                    parsed = current_parent.parsed
+                    parsed = current_parent.parsed_url
                     break
                 current_parent = current_parent.source
                 if current_parent.source.type == "SCAN":
diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py
index d801a755e..f23b89bf7 100644
--- a/bbot/modules/robots.py
+++ b/bbot/modules/robots.py
@@ -21,7 +21,7 @@ async def setup(self):
         return True
 
     async def handle_event(self, event):
-        host = f"{event.parsed.scheme}://{event.parsed.netloc}/"
+        host = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
         result = None
         url = f"{host}robots.txt"
         result = await self.helpers.request(url)
diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py
index d94a3b0a2..38a2b644f 100644
--- a/bbot/modules/secretsdb.py
+++ b/bbot/modules/secretsdb.py
@@ -51,7 +51,7 @@ async def handle_event(self, event):
             matches = [m.string[m.start() : m.end()] for m in matches]
             description = f"Possible secret ({name}): {matches}"
             event_data = {"host": str(event.host), "description": description}
-            parsed_url = getattr(event, "parsed", None)
+            parsed_url = getattr(event, "parsed_url", None)
             if parsed_url:
                 event_data["url"] = parsed_url.geturl()
             await self.emit_event(
diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py
index 74b702eaa..ca6ea1bad 100644
--- a/bbot/modules/url_manipulation.py
+++ b/bbot/modules/url_manipulation.py
@@ -94,10 +94,10 @@ async def filter_event(self, event):
 
     def format_signature(self, sig, event):
         if sig[2] == True:
-            cleaned_path = event.parsed.path.strip("/")
+            cleaned_path = event.parsed_url.path.strip("/")
         else:
-            cleaned_path = event.parsed.path.lstrip("/")
+            cleaned_path = event.parsed_url.path.lstrip("/")
 
-        kwargs = {"scheme": event.parsed.scheme, "netloc": event.parsed.netloc, "path": cleaned_path}
+        kwargs = {"scheme": event.parsed_url.scheme, "netloc": event.parsed_url.netloc, "path": cleaned_path}
         formatted_url = sig[1].format(**kwargs)
         return (sig[0], formatted_url)
diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py
index b8786e494..5cd6bc4e8 100644
--- a/bbot/modules/wafw00f.py
+++ b/bbot/modules/wafw00f.py
@@ -33,7 +33,7 @@ async def filter_event(self, event):
         return True, ""
 
     async def handle_event(self, event):
-        url = f"{event.parsed.scheme}://{event.parsed.netloc}/"
+        url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
         WW = await self.helpers.run_in_executor(wafw00f_main.WAFW00F, url, followredirect=False)
         waf_detections = await self.helpers.run_in_executor(WW.identwaf)
         if waf_detections:
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 011b174b8..dbea08f7f 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -72,8 +72,27 @@ async def handle_event(self, event, kwargs):
         # update event's scope distance based on its parent
         event.scope_distance = event.source.scope_distance + 1
 
-        # blacklist rejections
+        # special handling of URL extensions
+        parsed_url = getattr(event, "parsed_url", None)
+        if parsed_url is not None:
+            url_path = parsed_url.path
+            if url_path:
+                parsed_path_lower = str(url_path).lower()
+                extension = self.helpers.get_file_extension(parsed_path_lower)
+                if extension:
+                    event.add_tag(f"extension-{extension}")
+                if extension in self.scan.url_extension_httpx_only:
+                    event.add_tag("httpx-only")
+                    event._omit = True
+
+            # blacklist by extension
+            if extension in self.scan.url_extension_blacklist:
+                event.add_tag("blacklisted")
+
+        # main scan blacklist
         event_blacklisted = self.scan.blacklisted(event)
+
+        # reject all blacklisted events
         if event_blacklisted or "blacklisted" in event.tags:
             return False, f"Omitting blacklisted event: {event}"
 
diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py
index 75588b0eb..cb4e140e3 100644
--- a/bbot/scanner/preset/args.py
+++ b/bbot/scanner/preset/args.py
@@ -338,4 +338,4 @@ def validate(self):
                 if self.exclude_from_validation.match(c):
                     continue
                 # otherwise, ensure it exists as a module option
-                raise ValidationError(get_closest_match(c, all_options, msg="module option"))
+                raise ValidationError(get_closest_match(c, all_options, msg="config option"))
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 83118ac9e..db1382946 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -197,6 +197,7 @@ def __init__(
         self._finished_init = False
         self._new_activity = False
         self._cleanedup = False
+        self._omitted_event_types = None
 
         self.__loop = None
         self._manager_worker_loop_tasks = []
@@ -861,6 +862,12 @@ def aborting(self):
     def status(self):
         return self._status
 
+    @property
+    def omitted_event_types(self):
+        if self._omitted_event_types is None:
+            self._omitted_event_types = self.config.get("omit_event_types", [])
+        return self._omitted_event_types
+
     @status.setter
     def status(self, status):
         """
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 3b8ec29bf..d2cc2bf11 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -100,7 +100,7 @@ async def test_events(events, scan, helpers):
     # http response
     assert events.http_response.host == "example.com"
     assert events.http_response.port == 80
-    assert events.http_response.parsed.scheme == "http"
+    assert events.http_response.parsed_url.scheme == "http"
     assert events.http_response.with_port().geturl() == "http://example.com:80/"
 
     http_response = scan.make_event(
diff --git a/bbot/test/test_step_2/module_tests/test_module_dastardly.py b/bbot/test/test_step_2/module_tests/test_module_dastardly.py
index ed4c20e5c..6ab3a36a2 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dastardly.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dastardly.py
@@ -52,7 +52,7 @@ async def setup_after_prep(self, module_test):
         def new_filter_event(event):
             self.new_url = f"http://{docker_ip}:5556/"
             event.data["url"] = self.new_url
-            event.parsed = module_test.scan.helpers.urlparse(self.new_url)
+            event.parsed_url = module_test.scan.helpers.urlparse(self.new_url)
             return old_filter_event(event)
 
         module_test.monkeypatch.setattr(module_test.module, "filter_event", new_filter_event)

From 1d0ccf46d5d9014e13727418ae789c0a1b65342d Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 1 May 2024 13:20:38 +0100
Subject: [PATCH 040/220] Added description of the `FILESYSTEM` path

---
 bbot/modules/github_workflows.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 35609232c..8c8c2c35e 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -44,7 +44,7 @@ async def handle_event(self, event):
                 if log_path:
                     self.verbose(f"Downloaded repository workflow logs to {log_path}")
                     logfile_event = self.make_event(
-                        {"path": str(log_path)}, "FILESYSTEM", tags=["zipfile"], source=event
+                        {"path": str(log_path), "description": f"Workflow run logs from https://github.com/{owner}/{repo}/actions/runs/{run_id}"}, "FILESYSTEM", tags=["zipfile"], source=event
                     )
                     logfile_event.scope_distance = event.scope_distance
                     await self.emit_event(logfile_event)

From f4cec98faf89030c146741004bf2b59f486b1304 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 1 May 2024 13:21:44 +0100
Subject: [PATCH 041/220] Lint

---
 bbot/modules/github_workflows.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 8c8c2c35e..7c3150f68 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -44,7 +44,13 @@ async def handle_event(self, event):
                 if log_path:
                     self.verbose(f"Downloaded repository workflow logs to {log_path}")
                     logfile_event = self.make_event(
-                        {"path": str(log_path), "description": f"Workflow run logs from https://github.com/{owner}/{repo}/actions/runs/{run_id}"}, "FILESYSTEM", tags=["zipfile"], source=event
+                        {
+                            "path": str(log_path),
+                            "description": f"Workflow run logs from https://github.com/{owner}/{repo}/actions/runs/{run_id}",
+                        },
+                        "FILESYSTEM",
+                        tags=["zipfile"],
+                        source=event,
                     )
                     logfile_event.scope_distance = event.scope_distance
                     await self.emit_event(logfile_event)

From 7ed127eea91d89f0d045b2735bde540ad08d72c4 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 08:37:59 -0400
Subject: [PATCH 042/220] fix cli tests

---
 bbot/test/test_step_1/test_cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 5b4452b98..02d7d5b7b 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -325,7 +325,7 @@ def test_cli_config_validation(monkeypatch, caplog):
     assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-c", "modules.ipnegibhor.num_bits=4"])
     cli.main()
-    assert 'Could not find module option "modules.ipnegibhor.num_bits"' in caplog.text
+    assert 'Could not find config option "modules.ipnegibhor.num_bits"' in caplog.text
     assert 'Did you mean "modules.ipneighbor.num_bits"?' in caplog.text
 
     # incorrect global option
@@ -333,7 +333,7 @@ def test_cli_config_validation(monkeypatch, caplog):
     assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-c", "web_spier_distance=4"])
     cli.main()
-    assert 'Could not find module option "web_spier_distance"' in caplog.text
+    assert 'Could not find config option "web_spier_distance"' in caplog.text
     assert 'Did you mean "web_spider_distance"?' in caplog.text
 
 

From 6414d632d39e407d204f566bbbc9d53f971fdc1d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 08:47:01 -0400
Subject: [PATCH 043/220] update documentation

---
 bbot/core/helpers/dns/brute.py   | 7 ++++---
 bbot/core/helpers/wordcloud.py   | 2 +-
 bbot/defaults.yml                | 4 ++++
 docs/scanning/tips_and_tricks.md | 4 ++--
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
index d8996f2f4..c34e96610 100644
--- a/bbot/core/helpers/dns/brute.py
+++ b/bbot/core/helpers/dns/brute.py
@@ -23,7 +23,7 @@ def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.log = logging.getLogger("bbot.helper.dns.brute")
         self.num_canaries = 100
-        self.max_resolvers = 1000
+        self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
         self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
         self.digit_regex = self.parent_helper.re.compile(r"\d+")
         self._resolver_file = None
@@ -142,8 +142,9 @@ async def _massdns(self, module, domain, subdomains, rdtype):
 
     async def gen_subdomains(self, prefixes, domain):
         for p in prefixes:
-            d = f"{p}.{domain}"
-            yield d
+            if domain:
+                p = f"{p}.{domain}"
+            yield p
 
     async def resolver_file(self):
         if self._resolver_file is None:
diff --git a/bbot/core/helpers/wordcloud.py b/bbot/core/helpers/wordcloud.py
index 5eafb00c5..fbd4e7593 100644
--- a/bbot/core/helpers/wordcloud.py
+++ b/bbot/core/helpers/wordcloud.py
@@ -451,7 +451,7 @@ def add_word(self, word):
 
 class DNSMutator(Mutator):
     """
-    DNS-specific mutator used by the `massdns` module to generate target-specific subdomain mutations.
+    DNS-specific mutator used by the `dnsbrute_mutations` module to generate target-specific subdomain mutations.
 
     This class extends the Mutator base class to add DNS-specific logic for generating
     subdomain mutations based on input words. It utilizes custom word extraction patterns
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index 4b9b5210d..42cd265f0 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -24,6 +24,10 @@ deps:
   ffuf:
     version: "2.1.0"
 
+dns:
+  # Number of concurrent massdns lookups (-s)
+  brute_threads: 1000
+
 ### WEB SPIDER ###
 
 # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed)
diff --git a/docs/scanning/tips_and_tricks.md b/docs/scanning/tips_and_tricks.md
index 885e461dc..f019f742a 100644
--- a/docs/scanning/tips_and_tricks.md
+++ b/docs/scanning/tips_and_tricks.md
@@ -30,13 +30,13 @@ To change the number of instances, you can set a module's `max_event_handlers` i
 bbot -t evilcorp.com -m baddns -c modules.baddns.max_event_handlers=20
 ```
 
-### Boost Massdns Thread Count
+### Boost DNS Brute-force Speed
 
 If you have a fast internet connection or are running BBOT from a cloud VM, you can speed up subdomain enumeration by cranking the threads for `massdns`. The default is `1000`, which is about 1MB/s of DNS traffic:
 
 ```bash
 # massdns with 5000 resolvers, about 5MB/s
-bbot -t evilcorp.com -f subdomain-enum -c modules.massdns.max_resolvers=5000
+bbot -t evilcorp.com -f subdomain-enum -c dns.brute_threads=5000
 ```
 
 ### Web Spider

From 52a04fdfd36c6631e16b4f4a79908c7440291dd5 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 08:59:31 -0400
Subject: [PATCH 044/220] fix event tests

---
 bbot/core/event/base.py              | 15 +++++++++++++--
 bbot/scanner/manager.py              | 19 +++++++------------
 bbot/test/test_step_1/test_events.py | 11 +++++++++--
 3 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index abacbf132..ccdd4be1e 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -28,6 +28,7 @@
     split_host_port,
     tagify,
     validators,
+    get_file_extension,
 )
 
 
@@ -450,7 +451,7 @@ def get_source(self):
         """
         Takes into account events with the omit flag
         """
-        if getattr(self.source, "omit", False):
+        if getattr(self.source, "_omit", False):
             return self.source.get_source()
         return self.source
 
@@ -893,6 +894,16 @@ def __init__(self, *args, **kwargs):
     def sanitize_data(self, data):
         self.parsed_url = validators.validate_url_parsed(data)
 
+        # special handling of URL extensions
+        if self.parsed_url is not None:
+            url_path = self.parsed_url.path
+            if url_path:
+                parsed_path_lower = str(url_path).lower()
+                extension = get_file_extension(parsed_path_lower)
+                if extension:
+                    self.url_extension = extension
+                    self.add_tag(f"extension-{extension}")
+
         # tag as dir or endpoint
         if str(self.parsed_url.path).endswith("/"):
             self.add_tag("dir")
@@ -1042,7 +1053,7 @@ def redirect_location(self):
             # if there's no scheme (i.e. it's a relative redirect)
             if not scheme:
                 # then join the location with the current url
-                location = urljoin(self.parsed.geturl(), location)
+                location = urljoin(self.parsed_url.geturl(), location)
         return location
 
 
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index dbea08f7f..7c83e9205 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -73,20 +73,15 @@ async def handle_event(self, event, kwargs):
         event.scope_distance = event.source.scope_distance + 1
 
         # special handling of URL extensions
-        parsed_url = getattr(event, "parsed_url", None)
-        if parsed_url is not None:
-            url_path = parsed_url.path
-            if url_path:
-                parsed_path_lower = str(url_path).lower()
-                extension = self.helpers.get_file_extension(parsed_path_lower)
-                if extension:
-                    event.add_tag(f"extension-{extension}")
-                if extension in self.scan.url_extension_httpx_only:
-                    event.add_tag("httpx-only")
-                    event._omit = True
+        url_extension = getattr(event, "url_extension", None)
+        self.critical(f"{url_extension} in {self.scan.url_extension_httpx_only}?")
+        if url_extension is not None:
+            if url_extension in self.scan.url_extension_httpx_only:
+                event.add_tag("httpx-only")
+                event._omit = True
 
             # blacklist by extension
-            if extension in self.scan.url_extension_blacklist:
+            if url_extension in self.scan.url_extension_blacklist:
                 event.add_tag("blacklisted")
 
         # main scan blacklist
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index d2cc2bf11..986cc6db9 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -6,7 +6,13 @@
 
 
 @pytest.mark.asyncio
-async def test_events(events, scan, helpers):
+async def test_events(events, helpers):
+
+    from bbot.scanner import Scanner
+
+    scan = Scanner()
+    await scan._prep()
+
     assert events.ipv4.type == "IP_ADDRESS"
     assert events.ipv6.type == "IP_ADDRESS"
     assert events.netv4.type == "IP_RANGE"
@@ -159,8 +165,9 @@ async def test_events(events, scan, helpers):
     assert events.ipv6_url_unverified.host == ipaddress.ip_address("2001:4860:4860::8888")
     assert events.ipv6_url_unverified.port == 443
 
-    javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", dummy=True)
+    javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", source=scan.root_event)
     assert "extension-js" in javascript_event.tags
+    await scan.ingress_module.handle_event(javascript_event, {})
     assert "httpx-only" in javascript_event.tags
 
     # scope distance

From 856c23657b36370e2f938594098d7f268290c7b0 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 09:33:21 -0400
Subject: [PATCH 045/220] small tweak to error handling

---
 bbot/modules/github_workflows.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 7c3150f68..fb39a0283 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -110,14 +110,25 @@ async def download_run_logs(self, owner, repo, run_id):
         self.helpers.mkdir(folder)
         filename = f"run_{run_id}.zip"
         file_destination = folder / filename
-        result = await self.helpers.download(
-            f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs",
-            filename=file_destination,
-            headers=self.headers,
-        )
-        if result:
+        try:
+            result = await self.helpers.download(
+                f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs",
+                filename=file_destination,
+                headers=self.headers,
+                raise_error=True,
+                warn=False,
+            )
             self.info(f"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}")
             return file_destination
-        else:
-            self.warning(f"The logs for {owner}/{repo}/{run_id} have expired and are no longer available.")
-            return None
+        except Exception as e:
+            response = getattr(e, "response", None)
+            status_code = getattr(response, "status_code", 0)
+            if status_code == 403:
+                self.warning(
+                    f"The current access key does not have access to workflow {owner}/{repo}/{run_id} (status: {status_code})"
+                )
+            else:
+                self.info(
+                    f"The logs for {owner}/{repo}/{run_id} have expired and are no longer available (status: {status_code})"
+                )
+        return None

From 9e56987e678024eccdd99abd05bbddb8555282d3 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 09:36:55 -0400
Subject: [PATCH 046/220] flaked

---
 bbot/modules/github_workflows.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index fb39a0283..8999c41a3 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -111,7 +111,7 @@ async def download_run_logs(self, owner, repo, run_id):
         filename = f"run_{run_id}.zip"
         file_destination = folder / filename
         try:
-            result = await self.helpers.download(
+            await self.helpers.download(
                 f"{self.base_url}/repos/{owner}/{repo}/actions/runs/{run_id}/logs",
                 filename=file_destination,
                 headers=self.headers,

From 08de913b93b2c6dc244848ea14e8cf941795a199 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 10:36:56 -0400
Subject: [PATCH 047/220] fix event tests

---
 bbot/modules/output/base.py                 |  5 +-
 bbot/test/test_step_1/test_modules_basic.py | 63 +++++++++++++++++----
 2 files changed, 57 insertions(+), 11 deletions(-)

diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index 4a00af74d..a934ec009 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -41,6 +41,9 @@ def _event_precheck(self, event):
         if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
             return False, (f"Omitting {event} from output because it's marked as httpx-only")
 
+        if event._omit:
+            return False, "_omit is True"
+
         # omit certain event types
         if event.type in self.scan.omitted_event_types:
             if "target" in event.tags:
@@ -48,7 +51,7 @@ def _event_precheck(self, event):
             elif event.type in self.get_watched_events():
                 self.debug(f"Allowing omitted event: {event} because its type is explicitly in watched_events")
             else:
-                return False, f"its type is omitted in the config"
+                return False, "its type is omitted in the config"
 
         # internal events like those from speculate, ipneighbor
         # or events that are over our report distance
diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 03273c0a7..249953365 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -9,32 +9,75 @@
 
 
 @pytest.mark.asyncio
-async def test_modules_basic(scan, helpers, events, bbot_scanner, httpx_mock):
+async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     for http_method in ("GET", "CONNECT", "HEAD", "POST", "PUT", "TRACE", "DEBUG", "PATCH", "DELETE", "OPTIONS"):
         httpx_mock.add_response(method=http_method, url=re.compile(r".*"), json={"test": "test"})
 
+    from bbot.scanner import Scanner
+
+    scan = Scanner(config={"omit_event_types": ["URL_UNVERIFIED"]})
+    assert "URL_UNVERIFIED" in scan.omitted_event_types
+
     # output module specific event filtering tests
     base_output_module_1 = BaseOutputModule(scan)
-    base_output_module_1.watched_events = ["IP_ADDRESS"]
+    base_output_module_1.watched_events = ["IP_ADDRESS", "URL_UNVERIFIED"]
     localhost = scan.make_event("127.0.0.1", source=scan.root_event)
-    assert base_output_module_1._event_precheck(localhost)[0] == True
+    # ip addresses should be accepted
+    result, reason = base_output_module_1._event_precheck(localhost)
+    assert result == True
+    assert reason == "precheck succeeded"
+    # internal events should be rejected
     localhost._internal = True
-    assert base_output_module_1._event_precheck(localhost)[0] == False
+    result, reason = base_output_module_1._event_precheck(localhost)
+    assert result == False
+    assert reason == "_internal is True"
     localhost._internal = False
-    assert base_output_module_1._event_precheck(localhost)[0] == True
+    result, reason = base_output_module_1._event_precheck(localhost)
+    assert result == True
+    assert reason == "precheck succeeded"
+    # omitted events should be rejected
     localhost._omit = True
-    assert base_output_module_1._event_precheck(localhost)[0] == True
+    result, reason = base_output_module_1._event_precheck(localhost)
+    assert result == False
+    assert reason == "_omit is True"
+    # unwatched event types should be rejected
+    dns_name = scan.make_event("evilcorp.com", "DNS_NAME", source=scan.root_event)
+    result, reason = base_output_module_1._event_precheck(dns_name)
+    assert result == False
+    assert reason == "its type is not in watched_events"
+    # omitted event types matching watched events should be accepted
+    url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", source=scan.root_event)
+    result, reason = base_output_module_1._event_precheck(url_unverified)
+    assert result == True
+    assert reason == "precheck succeeded"
 
     base_output_module_2 = BaseOutputModule(scan)
     base_output_module_2.watched_events = ["*"]
+    # normal events should be accepted
     localhost = scan.make_event("127.0.0.1", source=scan.root_event)
-    assert base_output_module_2._event_precheck(localhost)[0] == True
+    result, reason = base_output_module_2._event_precheck(localhost)
+    assert result == True
+    assert reason == "precheck succeeded"
+    # internal events should be rejected
     localhost._internal = True
-    assert base_output_module_2._event_precheck(localhost)[0] == False
+    result, reason = base_output_module_2._event_precheck(localhost)
+    assert result == False
+    assert reason == "_internal is True"
     localhost._internal = False
-    assert base_output_module_2._event_precheck(localhost)[0] == True
+    result, reason = base_output_module_2._event_precheck(localhost)
+    assert result == True
+    assert reason == "precheck succeeded"
+    # omitted events should be rejected
     localhost._omit = True
-    assert base_output_module_2._event_precheck(localhost)[0] == False
+    result, reason = base_output_module_2._event_precheck(localhost)
+    assert result == False
+    assert reason == "_omit is True"
+    # omitted event types should be rejected
+    url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", source=scan.root_event)
+    result, reason = base_output_module_2._event_precheck(url_unverified)
+    log.critical(f"{url_unverified} / {result} / {reason}")
+    assert result == False
+    assert reason == "its type is omitted in the config"
 
     # common event filtering tests
     for module_class in (BaseModule, BaseOutputModule, BaseReportModule, BaseInternalModule):

From f61232f927dfd96454958d37272609071862d5ff Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 11:08:11 -0400
Subject: [PATCH 048/220] fix module sequence

---
 bbot/core/event/base.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ccdd4be1e..fc8cd3fdd 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -449,7 +449,7 @@ def source_id(self):
 
     def get_source(self):
         """
-        Takes into account events with the omit flag
+        Takes into account events with the _omit flag
         """
         if getattr(self.source, "_omit", False):
             return self.source.get_source()
@@ -665,7 +665,7 @@ def module_sequence(self):
             str: The module sequence in human-friendly format.
         """
         module_name = getattr(self.module, "name", "")
-        if getattr(self.source, "omit", False):
+        if getattr(self.source, "_omit", False):
             module_name = f"{self.source.module_sequence}->{module_name}"
         return module_name
 

From a0aaf5feba545bcd0733287568788bdfb5ae0bc6 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 1 May 2024 16:53:21 +0100
Subject: [PATCH 049/220] Add the `FILESYSTEM` event type to trufflehog if it
 is neither a git or docker repo

---
 bbot/modules/trufflehog.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index 0bf27b413..bc0b2b230 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -9,7 +9,7 @@ class trufflehog(BaseModule):
     meta = {"description": "TruffleHog is a tool for finding credentials"}
 
     options = {
-        "version": "3.69.0",
+        "version": "3.74.0",
         "only_verified": True,
         "concurrency": 8,
     }
@@ -37,18 +37,15 @@ async def setup(self):
         self.concurrency = int(self.config.get("concurrency", 8))
         return True
 
-    async def filter_event(self, event):
-        if event.type == "FILESYSTEM":
-            if "git" not in event.tags and "docker" not in event.tags:
-                return False, "event is not a git repository or a docker image"
-        return True
-
     async def handle_event(self, event):
         path = event.data["path"]
+        description = event.data.get("description", "")
         if "git" in event.tags:
             module = "git"
         elif "docker" in event.tags:
             module = "docker"
+        else:
+            module = "filesystem"
         async for decoder_name, detector_name, raw_result, verified, source_metadata in self.execute_trufflehog(
             module, path
         ):
@@ -58,12 +55,16 @@ async def handle_event(self, event):
                     "description": f"Verified Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
                     "host": str(event.source.host),
                 }
+                if description:
+                    data["description"] += f" Description: [{description}]"
                 await self.emit_event(data, "VULNERABILITY", event)
             else:
                 data = {
                     "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
                     "host": str(event.source.host),
                 }
+                if description:
+                    data["description"] += f" Description: [{description}]"
                 await self.emit_event(data, "FINDING", event)
 
     async def execute_trufflehog(self, module, path):
@@ -80,7 +81,11 @@ async def execute_trufflehog(self, module, path):
         elif module == "docker":
             command.append("docker")
             command.append("--image=file://" + path)
+        elif module == "filesystem":
+            command.append("filesystem")
+            command.append(path)
 
+        emitted_raw_results = set()
         stats_file = self.helpers.tempfile_tail(callback=self.log_trufflehog_status)
         try:
             with open(stats_file, "w") as stats_fh:
@@ -101,6 +106,10 @@ async def execute_trufflehog(self, module, path):
 
                     source_metadata = j.get("SourceMetadata", {})
 
+                    if raw_result in emitted_raw_results:
+                        continue
+
+                    emitted_raw_results.add(raw_result)
                     yield (decoder_name, detector_name, raw_result, verified, source_metadata)
         finally:
             stats_file.unlink()

From 50e283c64fdced28de57dac5a901817d1fe01775 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 1 May 2024 16:53:44 +0100
Subject: [PATCH 050/220] Add a module to test the modifications to trufflehog

---
 .../module_tests/test_module_trufflehog.py    | 410 +++++++++++++++++-
 1 file changed, 404 insertions(+), 6 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
index bfbe131ff..827f76d6f 100644
--- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
+++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
@@ -1,6 +1,7 @@
 import subprocess
 import shutil
 import io
+import zipfile
 import tarfile
 from pathlib import Path
 
@@ -8,7 +9,15 @@
 
 
 class TestTrufflehog(ModuleTestBase):
-    modules_overrides = ["github_org", "speculate", "git_clone", "dockerhub", "docker_pull", "trufflehog"]
+    modules_overrides = [
+        "github_org",
+        "speculate",
+        "git_clone",
+        "github_workflows",
+        "dockerhub",
+        "docker_pull",
+        "trufflehog",
+    ]
 
     file_content = "Verifyable Secret:\nhttps://admin:admin@the-internet.herokuapp.com/basic_auth\n\nUnverifyable Secret:\nhttps://admin:admin@internal.host.com"
 
@@ -152,9 +161,392 @@ async def setup_before_prep(self, module_test):
                     "watchers": 2,
                     "default_branch": "main",
                     "permissions": {"admin": False, "maintain": False, "push": False, "triage": False, "pull": True},
-                }
+                },
+                {
+                    "id": 459780477,
+                    "node_id": "R_kgDOG2exfQ",
+                    "name": "bbot",
+                    "full_name": "blacklanternsecurity/bbot",
+                    "private": False,
+                    "owner": {
+                        "login": "blacklanternsecurity",
+                        "id": 79229934,
+                        "node_id": "MDEyOk9yZ2FuaXphdGlvbjc5MjI5OTM0",
+                        "avatar_url": "https://avatars.githubusercontent.com/u/79229934?v=4",
+                        "gravatar_id": "",
+                        "url": "https://api.github.com/users/blacklanternsecurity",
+                        "html_url": "https://github.com/blacklanternsecurity",
+                        "followers_url": "https://api.github.com/users/blacklanternsecurity/followers",
+                        "following_url": "https://api.github.com/users/blacklanternsecurity/following{/other_user}",
+                        "gists_url": "https://api.github.com/users/blacklanternsecurity/gists{/gist_id}",
+                        "starred_url": "https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}",
+                        "subscriptions_url": "https://api.github.com/users/blacklanternsecurity/subscriptions",
+                        "organizations_url": "https://api.github.com/users/blacklanternsecurity/orgs",
+                        "repos_url": "https://api.github.com/users/blacklanternsecurity/repos",
+                        "events_url": "https://api.github.com/users/blacklanternsecurity/events{/privacy}",
+                        "received_events_url": "https://api.github.com/users/blacklanternsecurity/received_events",
+                        "type": "Organization",
+                        "site_admin": False,
+                    },
+                    "html_url": "https://github.com/blacklanternsecurity/bbot",
+                    "description": None,
+                    "fork": False,
+                    "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                    "forks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/forks",
+                    "keys_url": "https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}",
+                    "collaborators_url": "https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}",
+                    "teams_url": "https://api.github.com/repos/blacklanternsecurity/bbot/teams",
+                    "hooks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/hooks",
+                    "issue_events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}",
+                    "events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/events",
+                    "assignees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}",
+                    "branches_url": "https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}",
+                    "tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/tags",
+                    "blobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}",
+                    "git_tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}",
+                    "git_refs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}",
+                    "trees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}",
+                    "statuses_url": "https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}",
+                    "languages_url": "https://api.github.com/repos/blacklanternsecurity/bbot/languages",
+                    "stargazers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/stargazers",
+                    "contributors_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contributors",
+                    "subscribers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscribers",
+                    "subscription_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscription",
+                    "commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}",
+                    "git_commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}",
+                    "comments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}",
+                    "issue_comment_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}",
+                    "contents_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}",
+                    "compare_url": "https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}",
+                    "merges_url": "https://api.github.com/repos/blacklanternsecurity/bbot/merges",
+                    "archive_url": "https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}",
+                    "downloads_url": "https://api.github.com/repos/blacklanternsecurity/bbot/downloads",
+                    "issues_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}",
+                    "pulls_url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}",
+                    "milestones_url": "https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}",
+                    "notifications_url": "https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}",
+                    "labels_url": "https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}",
+                    "releases_url": "https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}",
+                    "deployments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/deployments",
+                    "created_at": "2022-02-15T23:10:51Z",
+                    "updated_at": "2023-09-02T12:20:13Z",
+                    "pushed_at": "2023-10-19T02:56:46Z",
+                    "git_url": "git://github.com/blacklanternsecurity/bbot.git",
+                    "ssh_url": "git@github.com:blacklanternsecurity/bbot.git",
+                    "clone_url": "https://github.com/blacklanternsecurity/bbot.git",
+                    "svn_url": "https://github.com/blacklanternsecurity/bbot",
+                    "homepage": None,
+                    "size": 2,
+                    "stargazers_count": 2,
+                    "watchers_count": 2,
+                    "language": None,
+                    "has_issues": True,
+                    "has_projects": True,
+                    "has_downloads": True,
+                    "has_wiki": True,
+                    "has_pages": False,
+                    "has_discussions": False,
+                    "forks_count": 32,
+                    "mirror_url": None,
+                    "archived": False,
+                    "disabled": False,
+                    "open_issues_count": 2,
+                    "license": None,
+                    "allow_forking": True,
+                    "is_template": False,
+                    "web_commit_signoff_required": False,
+                    "topics": [],
+                    "visibility": "public",
+                    "forks": 32,
+                    "open_issues": 2,
+                    "watchers": 2,
+                    "default_branch": "main",
+                    "permissions": {"admin": False, "maintain": False, "push": False, "triage": False, "pull": True},
+                },
             ],
         )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows?per_page=100&page=1",
+            json={
+                "total_count": 3,
+                "workflows": [
+                    {
+                        "id": 22452226,
+                        "node_id": "W_kwDOG_O3ns4BVpgC",
+                        "name": "tests",
+                        "path": ".github/workflows/tests.yml",
+                        "state": "active",
+                        "created_at": "2022-03-23T15:09:22.000Z",
+                        "updated_at": "2022-09-27T17:49:34.000Z",
+                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226",
+                        "html_url": "https://github.com/blacklanternsecurity/bbot/blob/stable/.github/workflows/tests.yml",
+                        "badge_url": "https://github.com/blacklanternsecurity/bbot/workflows/tests/badge.svg",
+                    },
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226/runs?status=success&per_page=1",
+            json={
+                "total_count": 2993,
+                "workflow_runs": [
+                    {
+                        "id": 8839360698,
+                        "name": "tests",
+                        "node_id": "WFR_kwLOG_O3ns8AAAACDt3wug",
+                        "head_branch": "dnsbrute-helperify",
+                        "head_sha": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                        "path": ".github/workflows/tests.yml",
+                        "display_title": "Helperify Massdns",
+                        "run_number": 4520,
+                        "event": "pull_request",
+                        "status": "completed",
+                        "conclusion": "success",
+                        "workflow_id": 22452226,
+                        "check_suite_id": 23162098295,
+                        "check_suite_node_id": "CS_kwDOG_O3ns8AAAAFZJGSdw",
+                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698",
+                        "html_url": "https://github.com/blacklanternsecurity/bbot/actions/runs/8839360698",
+                        "pull_requests": [
+                            {
+                                "url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls/1303",
+                                "id": 1839332952,
+                                "number": 1303,
+                                "head": {
+                                    "ref": "dnsbrute-helperify",
+                                    "sha": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                                    "repo": {
+                                        "id": 468957086,
+                                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                                        "name": "bbot",
+                                    },
+                                },
+                                "base": {
+                                    "ref": "faster-regexes",
+                                    "sha": "7baf219c7f3a4ba165639c5ddb62322453a8aea8",
+                                    "repo": {
+                                        "id": 468957086,
+                                        "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                                        "name": "bbot",
+                                    },
+                                },
+                            }
+                        ],
+                        "created_at": "2024-04-25T21:04:32Z",
+                        "updated_at": "2024-04-25T21:19:43Z",
+                        "actor": {
+                            "login": "TheTechromancer",
+                            "id": 20261699,
+                            "node_id": "MDQ6VXNlcjIwMjYxNjk5",
+                            "avatar_url": "https://avatars.githubusercontent.com/u/20261699?v=4",
+                            "gravatar_id": "",
+                            "url": "https://api.github.com/users/TheTechromancer",
+                            "html_url": "https://github.com/TheTechromancer",
+                            "followers_url": "https://api.github.com/users/TheTechromancer/followers",
+                            "following_url": "https://api.github.com/users/TheTechromancer/following{/other_user}",
+                            "gists_url": "https://api.github.com/users/TheTechromancer/gists{/gist_id}",
+                            "starred_url": "https://api.github.com/users/TheTechromancer/starred{/owner}{/repo}",
+                            "subscriptions_url": "https://api.github.com/users/TheTechromancer/subscriptions",
+                            "organizations_url": "https://api.github.com/users/TheTechromancer/orgs",
+                            "repos_url": "https://api.github.com/users/TheTechromancer/repos",
+                            "events_url": "https://api.github.com/users/TheTechromancer/events{/privacy}",
+                            "received_events_url": "https://api.github.com/users/TheTechromancer/received_events",
+                            "type": "User",
+                            "site_admin": False,
+                        },
+                        "run_attempt": 1,
+                        "referenced_workflows": [],
+                        "run_started_at": "2024-04-25T21:04:32Z",
+                        "triggering_actor": {
+                            "login": "TheTechromancer",
+                            "id": 20261699,
+                            "node_id": "MDQ6VXNlcjIwMjYxNjk5",
+                            "avatar_url": "https://avatars.githubusercontent.com/u/20261699?v=4",
+                            "gravatar_id": "",
+                            "url": "https://api.github.com/users/TheTechromancer",
+                            "html_url": "https://github.com/TheTechromancer",
+                            "followers_url": "https://api.github.com/users/TheTechromancer/followers",
+                            "following_url": "https://api.github.com/users/TheTechromancer/following{/other_user}",
+                            "gists_url": "https://api.github.com/users/TheTechromancer/gists{/gist_id}",
+                            "starred_url": "https://api.github.com/users/TheTechromancer/starred{/owner}{/repo}",
+                            "subscriptions_url": "https://api.github.com/users/TheTechromancer/subscriptions",
+                            "organizations_url": "https://api.github.com/users/TheTechromancer/orgs",
+                            "repos_url": "https://api.github.com/users/TheTechromancer/repos",
+                            "events_url": "https://api.github.com/users/TheTechromancer/events{/privacy}",
+                            "received_events_url": "https://api.github.com/users/TheTechromancer/received_events",
+                            "type": "User",
+                            "site_admin": False,
+                        },
+                        "jobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/jobs",
+                        "logs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/logs",
+                        "check_suite_url": "https://api.github.com/repos/blacklanternsecurity/bbot/check-suites/23162098295",
+                        "artifacts_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/artifacts",
+                        "cancel_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/cancel",
+                        "rerun_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/rerun",
+                        "previous_attempt_url": None,
+                        "workflow_url": "https://api.github.com/repos/blacklanternsecurity/bbot/actions/workflows/22452226",
+                        "head_commit": {
+                            "id": "c5de1360e8e5ccba04b23035f675a529282b7dc2",
+                            "tree_id": "fe9b345c0745a5bbacb806225e92e1c48fccf35c",
+                            "message": "remove debug message",
+                            "timestamp": "2024-04-25T21:02:37Z",
+                            "author": {"name": "TheTechromancer", "email": "thetechromancer@protonmail.com"},
+                            "committer": {"name": "TheTechromancer", "email": "thetechromancer@protonmail.com"},
+                        },
+                        "repository": {
+                            "id": 468957086,
+                            "node_id": "R_kgDOG_O3ng",
+                            "name": "bbot",
+                            "full_name": "blacklanternsecurity/bbot",
+                            "private": False,
+                            "owner": {
+                                "login": "blacklanternsecurity",
+                                "id": 25311592,
+                                "node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky",
+                                "avatar_url": "https://avatars.githubusercontent.com/u/25311592?v=4",
+                                "gravatar_id": "",
+                                "url": "https://api.github.com/users/blacklanternsecurity",
+                                "html_url": "https://github.com/blacklanternsecurity",
+                                "followers_url": "https://api.github.com/users/blacklanternsecurity/followers",
+                                "following_url": "https://api.github.com/users/blacklanternsecurity/following{/other_user}",
+                                "gists_url": "https://api.github.com/users/blacklanternsecurity/gists{/gist_id}",
+                                "starred_url": "https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}",
+                                "subscriptions_url": "https://api.github.com/users/blacklanternsecurity/subscriptions",
+                                "organizations_url": "https://api.github.com/users/blacklanternsecurity/orgs",
+                                "repos_url": "https://api.github.com/users/blacklanternsecurity/repos",
+                                "events_url": "https://api.github.com/users/blacklanternsecurity/events{/privacy}",
+                                "received_events_url": "https://api.github.com/users/blacklanternsecurity/received_events",
+                                "type": "Organization",
+                                "site_admin": False,
+                            },
+                            "html_url": "https://github.com/blacklanternsecurity/bbot",
+                            "description": "A recursive internet scanner for hackers.",
+                            "fork": False,
+                            "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                            "forks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/forks",
+                            "keys_url": "https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}",
+                            "collaborators_url": "https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}",
+                            "teams_url": "https://api.github.com/repos/blacklanternsecurity/bbot/teams",
+                            "hooks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/hooks",
+                            "issue_events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}",
+                            "events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/events",
+                            "assignees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}",
+                            "branches_url": "https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}",
+                            "tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/tags",
+                            "blobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}",
+                            "git_tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}",
+                            "git_refs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}",
+                            "trees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}",
+                            "statuses_url": "https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}",
+                            "languages_url": "https://api.github.com/repos/blacklanternsecurity/bbot/languages",
+                            "stargazers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/stargazers",
+                            "contributors_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contributors",
+                            "subscribers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscribers",
+                            "subscription_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscription",
+                            "commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}",
+                            "git_commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}",
+                            "comments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}",
+                            "issue_comment_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}",
+                            "contents_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}",
+                            "compare_url": "https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}",
+                            "merges_url": "https://api.github.com/repos/blacklanternsecurity/bbot/merges",
+                            "archive_url": "https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}",
+                            "downloads_url": "https://api.github.com/repos/blacklanternsecurity/bbot/downloads",
+                            "issues_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}",
+                            "pulls_url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}",
+                            "milestones_url": "https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}",
+                            "notifications_url": "https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}",
+                            "labels_url": "https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}",
+                            "releases_url": "https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}",
+                            "deployments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/deployments",
+                        },
+                        "head_repository": {
+                            "id": 468957086,
+                            "node_id": "R_kgDOG_O3ng",
+                            "name": "bbot",
+                            "full_name": "blacklanternsecurity/bbot",
+                            "private": False,
+                            "owner": {
+                                "login": "blacklanternsecurity",
+                                "id": 25311592,
+                                "node_id": "MDEyOk9yZ2FuaXphdGlvbjI1MzExNTky",
+                                "avatar_url": "https://avatars.githubusercontent.com/u/25311592?v=4",
+                                "gravatar_id": "",
+                                "url": "https://api.github.com/users/blacklanternsecurity",
+                                "html_url": "https://github.com/blacklanternsecurity",
+                                "followers_url": "https://api.github.com/users/blacklanternsecurity/followers",
+                                "following_url": "https://api.github.com/users/blacklanternsecurity/following{/other_user}",
+                                "gists_url": "https://api.github.com/users/blacklanternsecurity/gists{/gist_id}",
+                                "starred_url": "https://api.github.com/users/blacklanternsecurity/starred{/owner}{/repo}",
+                                "subscriptions_url": "https://api.github.com/users/blacklanternsecurity/subscriptions",
+                                "organizations_url": "https://api.github.com/users/blacklanternsecurity/orgs",
+                                "repos_url": "https://api.github.com/users/blacklanternsecurity/repos",
+                                "events_url": "https://api.github.com/users/blacklanternsecurity/events{/privacy}",
+                                "received_events_url": "https://api.github.com/users/blacklanternsecurity/received_events",
+                                "type": "Organization",
+                                "site_admin": False,
+                            },
+                            "html_url": "https://github.com/blacklanternsecurity/bbot",
+                            "description": "A recursive internet scanner for hackers.",
+                            "fork": False,
+                            "url": "https://api.github.com/repos/blacklanternsecurity/bbot",
+                            "forks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/forks",
+                            "keys_url": "https://api.github.com/repos/blacklanternsecurity/bbot/keys{/key_id}",
+                            "collaborators_url": "https://api.github.com/repos/blacklanternsecurity/bbot/collaborators{/collaborator}",
+                            "teams_url": "https://api.github.com/repos/blacklanternsecurity/bbot/teams",
+                            "hooks_url": "https://api.github.com/repos/blacklanternsecurity/bbot/hooks",
+                            "issue_events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/events{/number}",
+                            "events_url": "https://api.github.com/repos/blacklanternsecurity/bbot/events",
+                            "assignees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/assignees{/user}",
+                            "branches_url": "https://api.github.com/repos/blacklanternsecurity/bbot/branches{/branch}",
+                            "tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/tags",
+                            "blobs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/blobs{/sha}",
+                            "git_tags_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/tags{/sha}",
+                            "git_refs_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/refs{/sha}",
+                            "trees_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/trees{/sha}",
+                            "statuses_url": "https://api.github.com/repos/blacklanternsecurity/bbot/statuses/{sha}",
+                            "languages_url": "https://api.github.com/repos/blacklanternsecurity/bbot/languages",
+                            "stargazers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/stargazers",
+                            "contributors_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contributors",
+                            "subscribers_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscribers",
+                            "subscription_url": "https://api.github.com/repos/blacklanternsecurity/bbot/subscription",
+                            "commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/commits{/sha}",
+                            "git_commits_url": "https://api.github.com/repos/blacklanternsecurity/bbot/git/commits{/sha}",
+                            "comments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/comments{/number}",
+                            "issue_comment_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues/comments{/number}",
+                            "contents_url": "https://api.github.com/repos/blacklanternsecurity/bbot/contents/{+path}",
+                            "compare_url": "https://api.github.com/repos/blacklanternsecurity/bbot/compare/{base}...{head}",
+                            "merges_url": "https://api.github.com/repos/blacklanternsecurity/bbot/merges",
+                            "archive_url": "https://api.github.com/repos/blacklanternsecurity/bbot/{archive_format}{/ref}",
+                            "downloads_url": "https://api.github.com/repos/blacklanternsecurity/bbot/downloads",
+                            "issues_url": "https://api.github.com/repos/blacklanternsecurity/bbot/issues{/number}",
+                            "pulls_url": "https://api.github.com/repos/blacklanternsecurity/bbot/pulls{/number}",
+                            "milestones_url": "https://api.github.com/repos/blacklanternsecurity/bbot/milestones{/number}",
+                            "notifications_url": "https://api.github.com/repos/blacklanternsecurity/bbot/notifications{?since,all,participating}",
+                            "labels_url": "https://api.github.com/repos/blacklanternsecurity/bbot/labels{/name}",
+                            "releases_url": "https://api.github.com/repos/blacklanternsecurity/bbot/releases{/id}",
+                            "deployments_url": "https://api.github.com/repos/blacklanternsecurity/bbot/deployments",
+                        },
+                    },
+                ],
+            },
+        )
+        module_test.httpx_mock.add_response(
+            url="https://api.github.com/repos/blacklanternsecurity/bbot/actions/runs/8839360698/logs",
+            headers={
+                "location": "https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02"
+            },
+            status_code=302,
+        )
+        data = io.BytesIO()
+        with zipfile.ZipFile(data, mode="w", compression=zipfile.ZIP_DEFLATED) as z:
+            z.writestr("test.txt", self.file_content)
+        data.seek(0)
+        zip_content = data.getvalue()
+        module_test.httpx_mock.add_response(
+            url="https://productionresultssa10.blob.core.windows.net/actions-results/7beb304e-f42c-4830-a027-4f5dec53107d/workflow-job-run-3a559e2a-952e-58d2-b8db-2e604a9266d7/logs/steps/step-logs-0e34a19a-18b0-4208-b27a-f8c031db2d17.txt?rsct=text%2Fplain&se=2024-04-26T16%3A25%3A39Z&sig=a%2FiN8dOw0e3tiBQZAfr80veI8OYChb9edJ1eFY136B4%3D&sp=r&spr=https&sr=b&st=2024-04-26T16%3A15%3A34Z&sv=2021-12-02",
+            content=zip_content,
+        )
         module_test.httpx_mock.add_response(
             url="https://hub.docker.com/v2/users/blacklanternsecurity",
             json={
@@ -460,13 +852,16 @@ def check(self, module_test, events):
             and "Verified Secret Found." in e.data["description"]
             and "Secret: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"]
         ]
-        assert 2 == len(vuln_events), "Failed to find secret in events"
-        github_repo_event = [e for e in vuln_events if e.data["host"] == "github.com"][0].source
+        assert 3 == len(vuln_events), "Failed to find secret in events"
+        github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].source
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
+        github_workflow_event = [e for e in vuln_events if "bbot" in e.data["description"]][0].source
+        file = Path(github_workflow_event.data["path"])
+        assert file.is_file(), "Destination zip does not exist"
         docker_source_event = [e for e in vuln_events if e.data["host"] == "hub.docker.com"][0].source
         file = Path(docker_source_event.data["path"])
         assert file.is_file(), "Destination image does not exist"
@@ -484,13 +879,16 @@ def check(self, module_test, events):
             and "Potential Secret Found." in e.data["description"]
             and "Secret: [https://admin:admin@internal.host.com]" in e.data["description"]
         ]
-        assert 2 == len(finding_events), "Failed to find secret in events"
-        github_repo_event = [e for e in finding_events if e.data["host"] == "github.com"][0].source
+        assert 3 == len(finding_events), "Failed to find secret in events"
+        github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].source
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
+        github_workflow_event = [e for e in finding_events if "bbot" in e.data["description"]][0].source
+        file = Path(github_workflow_event.data["path"])
+        assert file.is_file(), "Destination zip does not exist"
         docker_source_event = [e for e in finding_events if e.data["host"] == "hub.docker.com"][0].source
         file = Path(docker_source_event.data["path"])
         assert file.is_file(), "Destination image does not exist"

From 629a047016387b49c4cfdfb776bc0791b4b59721 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 1 May 2024 16:54:19 +0100
Subject: [PATCH 051/220] Add a description to docker containers to make them
 the alerts more userfriendly

---
 bbot/modules/docker_pull.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py
index bcb731f4d..4515e6d4a 100644
--- a/bbot/modules/docker_pull.py
+++ b/bbot/modules/docker_pull.py
@@ -50,7 +50,10 @@ async def handle_event(self, event):
         if repo_path:
             self.verbose(f"Downloaded docker repository {repo_url} to {repo_path}")
             codebase_event = self.make_event(
-                {"path": str(repo_path)}, "FILESYSTEM", tags=["docker", "tarball"], source=event
+                {"path": str(repo_path), "description": f"Docker image: {repo_url}"},
+                "FILESYSTEM",
+                tags=["docker", "tarball"],
+                source=event,
             )
             codebase_event.scope_distance = event.scope_distance
             await self.emit_event(codebase_event)

From 38fb669d6080d8f32e3de8dc8532fc0e79f9b326 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 12:27:52 -0400
Subject: [PATCH 052/220] tests for module_sequence

---
 bbot/test/test_step_1/test_events.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 986cc6db9..f831391be 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -467,3 +467,28 @@ async def test_events(events, helpers):
     )
     assert bucket_event.data["name"] == "asdf.s3.amazonaws.com"
     assert bucket_event.data["url"] == "https://asdf.s3.amazonaws.com/"
+
+    # test module sequence
+    module = scan._make_dummy_module("mymodule")
+    source_event_1 = scan.make_event("127.0.0.1", module=module, source=scan.root_event)
+    assert str(source_event_1.module) == "mymodule"
+    assert str(source_event_1.module_sequence) == "mymodule"
+    source_event_2 = scan.make_event("127.0.0.2", module=module, source=source_event_1)
+    assert str(source_event_2.module) == "mymodule"
+    assert str(source_event_2.module_sequence) == "mymodule"
+    source_event_3 = scan.make_event("127.0.0.3", module=module, source=source_event_2)
+    assert str(source_event_3.module) == "mymodule"
+    assert str(source_event_3.module_sequence) == "mymodule"
+
+    module = scan._make_dummy_module("mymodule")
+    source_event_1 = scan.make_event("127.0.0.1", module=module, source=scan.root_event)
+    source_event_1._omit = True
+    assert str(source_event_1.module) == "mymodule"
+    assert str(source_event_1.module_sequence) == "mymodule"
+    source_event_2 = scan.make_event("127.0.0.2", module=module, source=source_event_1)
+    source_event_2._omit = True
+    assert str(source_event_2.module) == "mymodule"
+    assert str(source_event_2.module_sequence) == "mymodule->mymodule"
+    source_event_3 = scan.make_event("127.0.0.3", module=module, source=source_event_2)
+    assert str(source_event_3.module) == "mymodule"
+    assert str(source_event_3.module_sequence) == "mymodule->mymodule->mymodule"

From 5c9c1b8ad429d1a36399e8c876507dbaf28880da Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 1 May 2024 23:58:34 -0400
Subject: [PATCH 053/220] target perf optimization

---
 bbot/modules/dnsbrute_mutations.py |  2 +-
 bbot/modules/nmap.py               |  2 +-
 bbot/scanner/target.py             | 32 +++++++++++++++++++-----------
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py
index 06fd03689..a02ef7e9e 100644
--- a/bbot/modules/dnsbrute_mutations.py
+++ b/bbot/modules/dnsbrute_mutations.py
@@ -104,7 +104,7 @@ def add_mutation(m):
                         self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(trimmed_found)})")
                         results = await self.helpers.dns.brute(self, query, mutations)
                         for hostname in results:
-                            source_event = self.source_events.get(hostname)
+                            source_event = self.source_events.get_host(hostname)
                             if source_event is None:
                                 self.warning(f"Could not correlate source event from: {hostname}")
                                 self.warning(self.source_events._radix.dns_tree.root.children)
diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py
index ccdb0974e..4374155ab 100644
--- a/bbot/modules/nmap.py
+++ b/bbot/modules/nmap.py
@@ -43,7 +43,7 @@ async def handle_batch(self, *events):
             for host in self.parse_nmap_xml(output_file):
                 source_event = None
                 for h in [host.address] + host.hostnames:
-                    source_event = target.get(h)
+                    source_event = target.get_host(h)
                     if source_event is not None:
                         break
                 if source_event is None:
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index 878e80846..9cca55ecc 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -7,6 +7,7 @@
 
 from bbot.errors import *
 from bbot.modules.base import BaseModule
+from bbot.core.helpers.misc import make_ip_type
 from bbot.core.event import make_event, is_event
 
 log = logging.getLogger("bbot.core.target")
@@ -212,20 +213,27 @@ def get(self, host):
         """
 
         try:
-            other = make_event(host, dummy=True)
+            event = make_event(host, dummy=True)
         except ValidationError:
             return
-        if other.host:
-            with suppress(KeyError, StopIteration):
-                result = self._radix.search(other.host)
-                if result is not None:
-                    for event in result:
-                        # if the result is a dns name and strict scope is enabled
-                        if isinstance(event.host, str) and self.strict_scope:
-                            # if the result doesn't exactly equal the host, abort
-                            if event.host != other.host:
-                                return
-                        return event
+        if event.host:
+            return self.get_host(event.host)
+
+    def get_host(self, host):
+        """
+        A more efficient version of .get() that only accepts hostnames and IP addresses
+        """
+        host = make_ip_type(host)
+        with suppress(KeyError, StopIteration):
+            result = self._radix.search(host)
+            if result is not None:
+                for event in result:
+                    # if the result is a dns name and strict scope is enabled
+                    if isinstance(event.host, str) and self.strict_scope:
+                        # if the result doesn't exactly equal the host, abort
+                        if event.host != host:
+                            return
+                    return event
 
     def _add_event(self, event):
         radix_data = self._radix.search(event.host)

From 1cf85cdf8eeaf7d08038d712746eb1276d523310 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 2 May 2024 00:41:40 -0400
Subject: [PATCH 054/220] write tests for url extension handling

---
 bbot/scanner/manager.py            |  6 ++++--
 bbot/test/test_step_1/test_scan.py | 22 ++++++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 7c83e9205..a1f8af0a2 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -74,7 +74,6 @@ async def handle_event(self, event, kwargs):
 
         # special handling of URL extensions
         url_extension = getattr(event, "url_extension", None)
-        self.critical(f"{url_extension} in {self.scan.url_extension_httpx_only}?")
         if url_extension is not None:
             if url_extension in self.scan.url_extension_httpx_only:
                 event.add_tag("httpx-only")
@@ -82,6 +81,9 @@ async def handle_event(self, event, kwargs):
 
             # blacklist by extension
             if url_extension in self.scan.url_extension_blacklist:
+                self.debug(
+                    f"Blacklisting {event} because its extension (.{url_extension}) is blacklisted in the config"
+                )
                 event.add_tag("blacklisted")
 
         # main scan blacklist
@@ -89,7 +91,7 @@ async def handle_event(self, event, kwargs):
 
         # reject all blacklisted events
         if event_blacklisted or "blacklisted" in event.tags:
-            return False, f"Omitting blacklisted event: {event}"
+            return False, "event is blacklisted"
 
         # Scope shepherding
         # here is where we make sure in-scope events are set to their proper scope distance
diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py
index e5648c4ee..a2bb68bbe 100644
--- a/bbot/test/test_step_1/test_scan.py
+++ b/bbot/test/test_step_1/test_scan.py
@@ -73,3 +73,25 @@ async def test_scan(
         events.append(event)
     event_data = [e.data for e in events]
     assert "one.one.one.one" not in event_data
+
+
+@pytest.mark.asyncio
+async def test_url_extension_handling(bbot_scanner):
+    scan = bbot_scanner(config={"url_extension_blacklist": ["css"], "url_extension_httpx_only": ["js"]})
+    await scan._prep()
+    assert scan.url_extension_blacklist == {"css"}
+    assert scan.url_extension_httpx_only == {"js"}
+    good_event = scan.make_event("https://evilcorp.com/a.txt", "URL", tags=["status-200"], source=scan.root_event)
+    bad_event = scan.make_event("https://evilcorp.com/a.css", "URL", tags=["status-200"], source=scan.root_event)
+    httpx_event = scan.make_event("https://evilcorp.com/a.js", "URL", tags=["status-200"], source=scan.root_event)
+    assert "blacklisted" not in bad_event.tags
+    assert "httpx-only" not in httpx_event.tags
+    result = await scan.ingress_module.handle_event(good_event, {})
+    assert result == None
+    result, reason = await scan.ingress_module.handle_event(bad_event, {})
+    assert result == False
+    assert reason == "event is blacklisted"
+    assert "blacklisted" in bad_event.tags
+    result = await scan.ingress_module.handle_event(httpx_event, {})
+    assert result == None
+    assert "httpx-only" in httpx_event.tags

From 886c189f26b3cc5b7af52191a255ebbdcd7afb10 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 2 May 2024 01:30:24 -0400
Subject: [PATCH 055/220] WIP http engine

---
 bbot/core/engine.py                |  64 +++++---
 bbot/core/helpers/web/__init__.py  |   1 +
 bbot/core/helpers/web/engine.py    | 225 +++++++++++++++++++++++++++++
 bbot/core/helpers/{ => web}/web.py | 218 ++--------------------------
 bbot/errors.py                     |   4 +
 bbot/test/test_step_1/test_web.py  |   7 +
 6 files changed, 286 insertions(+), 233 deletions(-)
 create mode 100644 bbot/core/helpers/web/__init__.py
 create mode 100644 bbot/core/helpers/web/engine.py
 rename bbot/core/helpers/{ => web}/web.py (74%)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 24781ab3b..2b31fa081 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -7,10 +7,12 @@
 import tempfile
 import traceback
 import zmq.asyncio
+import multiprocessing
 from pathlib import Path
 from contextlib import asynccontextmanager, suppress
 
 from bbot.core import CORE
+from bbot.errors import BBOTEngineError
 from bbot.core.helpers.misc import rand_string
 
 CMD_EXIT = 1000
@@ -22,6 +24,7 @@ class EngineClient:
 
     def __init__(self, **kwargs):
         self.name = f"EngineClient {self.__class__.__name__}"
+        self.process_name = multiprocessing.current_process().name
         if self.SERVER_CLASS is None:
             raise ValueError(f"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}")
         self.CMDS = dict(self.SERVER_CLASS.CMDS)
@@ -35,9 +38,9 @@ def __init__(self, **kwargs):
         self.context = zmq.asyncio.Context()
         atexit.register(self.cleanup)
 
-    async def run_and_return(self, command, **kwargs):
+    async def run_and_return(self, command, *args, **kwargs):
         async with self.new_socket() as socket:
-            message = self.make_message(command, args=kwargs)
+            message = self.make_message(command, args=args, kwargs=kwargs)
             await socket.send(message)
             binary = await socket.recv()
         # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}")
@@ -48,8 +51,8 @@ async def run_and_return(self, command, **kwargs):
             return
         return message
 
-    async def run_and_yield(self, command, **kwargs):
-        message = self.make_message(command, args=kwargs)
+    async def run_and_yield(self, command, *args, **kwargs):
+        message = self.make_message(command, args=args, kwargs=kwargs)
         async with self.new_socket() as socket:
             await socket.send(message)
             while 1:
@@ -75,28 +78,37 @@ def check_stop(self, message):
             return True
         return False
 
-    def make_message(self, command, args):
+    def make_message(self, command, args=None, kwargs=None):
         try:
             cmd_id = self.CMDS[command]
         except KeyError:
             raise KeyError(f'Command "{command}" not found. Available commands: {",".join(self.available_commands)}')
-        return pickle.dumps(dict(c=cmd_id, a=args))
+        message = {"c": cmd_id}
+        if args:
+            message["a"] = args
+        if kwargs:
+            message["k"] = kwargs
+        return pickle.dumps(message)
 
     @property
     def available_commands(self):
         return [s for s in self.CMDS if isinstance(s, str)]
 
     def start_server(self):
-        process = CORE.create_process(
-            target=self.server_process,
-            args=(
-                self.SERVER_CLASS,
-                self.socket_path,
-            ),
-            kwargs=self.server_kwargs,
-        )
-        process.start()
-        return process
+        self.log.critical(f"STARTING SERVER from {self.process_name}")
+        if self.process_name == "MainProcess":
+            process = CORE.create_process(
+                target=self.server_process,
+                args=(
+                    self.SERVER_CLASS,
+                    self.socket_path,
+                ),
+                kwargs=self.server_kwargs,
+            )
+            process.start()
+            return process
+        else:
+            raise BBOTEngineError(f"Tried to start server from process {self.process_name}")
 
     @staticmethod
     def server_process(server_class, socket_path, **kwargs):
@@ -145,20 +157,20 @@ def __init__(self, socket_path):
             # create socket file
             self.socket.bind(f"ipc://{socket_path}")
 
-    async def run_and_return(self, client_id, command_fn, **kwargs):
+    async def run_and_return(self, client_id, command_fn, *args, **kwargs):
         self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({kwargs})")
         try:
-            result = await command_fn(**kwargs)
+            result = await command_fn(*args, **kwargs)
         except Exception as e:
             error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
             trace = traceback.format_exc()
             result = {"_e": (error, trace)}
         await self.send_socket_multipart([client_id, pickle.dumps(result)])
 
-    async def run_and_yield(self, client_id, command_fn, **kwargs):
+    async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
         self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({kwargs})")
         try:
-            async for _ in command_fn(**kwargs):
+            async for _ in command_fn(*args, **kwargs):
                 await self.send_socket_multipart([client_id, pickle.dumps(_)])
             await self.send_socket_multipart([client_id, pickle.dumps({"_s": None})])
         except Exception as e:
@@ -186,9 +198,13 @@ async def worker(self):
                     self.log.warning(f"No command sent in message: {message}")
                     continue
 
-                kwargs = message.get("a", {})
+                args = message.get("a", ())
+                if not isinstance(args, tuple):
+                    self.log.warning(f"{self.name}: received invalid args of type {type(args)}, should be tuple")
+                    continue
+                kwargs = message.get("k", {})
                 if not isinstance(kwargs, dict):
-                    self.log.warning(f"{self.name}: received invalid message of type {type(kwargs)}, should be dict")
+                    self.log.warning(f"{self.name}: received invalid kwargs of type {type(kwargs)}, should be dict")
                     continue
 
                 command_name = self.CMDS[cmd]
@@ -199,9 +215,9 @@ async def worker(self):
                     continue
 
                 if inspect.isasyncgenfunction(command_fn):
-                    coroutine = self.run_and_yield(client_id, command_fn, **kwargs)
+                    coroutine = self.run_and_yield(client_id, command_fn, *args, **kwargs)
                 else:
-                    coroutine = self.run_and_return(client_id, command_fn, **kwargs)
+                    coroutine = self.run_and_return(client_id, command_fn, *args, **kwargs)
 
                 asyncio.create_task(coroutine)
         except Exception as e:
diff --git a/bbot/core/helpers/web/__init__.py b/bbot/core/helpers/web/__init__.py
new file mode 100644
index 000000000..8fcf82abb
--- /dev/null
+++ b/bbot/core/helpers/web/__init__.py
@@ -0,0 +1 @@
+from .web import WebHelper
diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
new file mode 100644
index 000000000..9e30dbb8b
--- /dev/null
+++ b/bbot/core/helpers/web/engine.py
@@ -0,0 +1,225 @@
+import ssl
+import httpx
+from httpx._models import Cookies
+from contextlib import asynccontextmanager
+
+from bbot.core.engine import EngineServer
+
+
+class DummyCookies(Cookies):
+    def extract_cookies(self, *args, **kwargs):
+        pass
+
+
+class BBOTAsyncClient(httpx.AsyncClient):
+    """
+    A subclass of httpx.AsyncClient tailored with BBOT-specific configurations and functionalities.
+    This class provides rate limiting, logging, configurable timeouts, user-agent customization, custom
+    headers, and proxy settings. Additionally, it allows the disabling of cookies, making it suitable
+    for use across an entire scan.
+
+    Attributes:
+        _bbot_scan (object): BBOT scan object containing configuration details.
+        _persist_cookies (bool): Flag to determine whether cookies should be persisted across requests.
+
+    Examples:
+        >>> async with BBOTAsyncClient(_bbot_scan=bbot_scan_object) as client:
+        >>>     response = await client.request("GET", "https://example.com")
+        >>>     print(response.status_code)
+        200
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._config = kwargs.pop("_config")
+        web_requests_per_second = self._config.get("web_requests_per_second", 100)
+
+        http_debug = self._config.get("http_debug", None)
+        if http_debug:
+            log.trace(f"Creating AsyncClient: {args}, {kwargs}")
+
+        self._persist_cookies = kwargs.pop("persist_cookies", True)
+
+        # timeout
+        http_timeout = self._config.get("http_timeout", 20)
+        if not "timeout" in kwargs:
+            kwargs["timeout"] = http_timeout
+
+        # headers
+        headers = kwargs.get("headers", None)
+        if headers is None:
+            headers = {}
+        # user agent
+        user_agent = self._config.get("user_agent", "BBOT")
+        if "User-Agent" not in headers:
+            headers["User-Agent"] = user_agent
+        kwargs["headers"] = headers
+        # proxy
+        proxies = self._config.get("http_proxy", None)
+        kwargs["proxies"] = proxies
+
+        super().__init__(*args, **kwargs)
+        if not self._persist_cookies:
+            self._cookies = DummyCookies()
+
+    def build_request(self, *args, **kwargs):
+        request = super().build_request(*args, **kwargs)
+        # add custom headers if the URL is in-scope
+        # TODO: re-enable this
+        # if self._preset.in_scope(str(request.url)):
+        #     for hk, hv in self._config.get("http_headers", {}).items():
+        #         # don't clobber headers
+        #         if hk not in request.headers:
+        #             request.headers[hk] = hv
+        return request
+
+    def _merge_cookies(self, cookies):
+        if self._persist_cookies:
+            return super()._merge_cookies(cookies)
+        return cookies
+
+
+class HTTPEngine(EngineServer):
+
+    CMDS = {
+        0: "request",
+        1: "request_batch",
+        2: "request_custom_batch",
+        99: "_mock",
+    }
+
+    client_only_options = (
+        "retries",
+        "max_redirects",
+    )
+
+    def __init__(self, socket_path, config={}):
+        super().__init__(socket_path)
+        self.log.critical("doing")
+        self.config = config
+        self.http_debug = self.config.get("http_debug", False)
+        self._ssl_context_noverify = None
+        self.ssl_verify = self.config.get("ssl_verify", False)
+        if self.ssl_verify is False:
+            self.ssl_verify = self.ssl_context_noverify()
+        self.web_client = self.AsyncClient(persist_cookies=False)
+
+    def AsyncClient(self, *args, **kwargs):
+        kwargs["_config"] = self.config
+        retries = kwargs.pop("retries", self.config.get("http_retries", 1))
+        kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=self.ssl_verify)
+        kwargs["verify"] = self.ssl_verify
+        return BBOTAsyncClient(*args, **kwargs)
+
+    async def request(self, *args, **kwargs):
+        self.log.critical(f"SERVER {args} / {kwargs}")
+        raise_error = kwargs.pop("raise_error", False)
+        # TODO: use this
+        cache_for = kwargs.pop("cache_for", None)  # noqa
+
+        client = kwargs.get("client", self.web_client)
+
+        # allow vs follow, httpx why??
+        allow_redirects = kwargs.pop("allow_redirects", None)
+        if allow_redirects is not None and "follow_redirects" not in kwargs:
+            kwargs["follow_redirects"] = allow_redirects
+
+        # in case of URL only, assume GET request
+        if len(args) == 1:
+            kwargs["url"] = args[0]
+            args = []
+
+        url = kwargs.get("url", "")
+
+        if not args and "method" not in kwargs:
+            kwargs["method"] = "GET"
+
+        client_kwargs = {}
+        for k in list(kwargs):
+            if k in self.client_only_options:
+                v = kwargs.pop(k)
+                client_kwargs[k] = v
+
+        if client_kwargs:
+            client = self.AsyncClient(**client_kwargs)
+
+        async with self._acatch(url, raise_error):
+            if self.http_debug:
+                logstr = f"Web request: {str(args)}, {str(kwargs)}"
+                log.trace(logstr)
+            response = await client.request(*args, **kwargs)
+            if self.http_debug:
+                log.trace(
+                    f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
+                )
+            return response
+
+    def ssl_context_noverify(self):
+        if self._ssl_context_noverify is None:
+            ssl_context = ssl.create_default_context()
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+            ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3
+            ssl_context.set_ciphers("ALL:@SECLEVEL=0")
+            ssl_context.options |= 0x4  # Add the OP_LEGACY_SERVER_CONNECT option
+            self._ssl_context_noverify = ssl_context
+        return self._ssl_context_noverify
+
+    @asynccontextmanager
+    async def _acatch(self, url, raise_error):
+        """
+        Asynchronous context manager to handle various httpx errors during a request.
+
+        Yields:
+            None
+
+        Note:
+            This function is internal and should generally not be used directly.
+            `url`, `args`, `kwargs`, and `raise_error` should be in the same context as this function.
+        """
+        try:
+            yield
+        except httpx.TimeoutException:
+            if raise_error:
+                raise
+            else:
+                log.verbose(f"HTTP timeout to URL: {url}")
+        except httpx.ConnectError:
+            if raise_error:
+                raise
+            else:
+                log.debug(f"HTTP connect failed to URL: {url}")
+        except httpx.HTTPError as e:
+            if raise_error:
+                raise
+            else:
+                log.trace(f"Error with request to URL: {url}: {e}")
+                log.trace(traceback.format_exc())
+        except ssl.SSLError as e:
+            msg = f"SSL error with request to URL: {url}: {e}"
+            if raise_error:
+                raise httpx.RequestError(msg)
+            else:
+                log.trace(msg)
+                log.trace(traceback.format_exc())
+        except anyio.EndOfStream as e:
+            msg = f"AnyIO error with request to URL: {url}: {e}"
+            if raise_error:
+                raise httpx.RequestError(msg)
+            else:
+                log.trace(msg)
+                log.trace(traceback.format_exc())
+        except SOCKSError as e:
+            msg = f"SOCKS error with request to URL: {url}: {e}"
+            if raise_error:
+                raise httpx.RequestError(msg)
+            else:
+                log.trace(msg)
+                log.trace(traceback.format_exc())
+        except BaseException as e:
+            # don't log if the error is the result of an intentional cancellation
+            if not any(
+                isinstance(_e, asyncio.exceptions.CancelledError) for _e in self.parent_helper.get_exception_chain(e)
+            ):
+                log.trace(f"Unhandled exception with request to URL: {url}: {e}")
+                log.trace(traceback.format_exc())
+            raise
diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web/web.py
similarity index 74%
rename from bbot/core/helpers/web.py
rename to bbot/core/helpers/web/web.py
index 26773bc9c..6d44cca61 100644
--- a/bbot/core/helpers/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -1,5 +1,4 @@
 import re
-import ssl
 import anyio
 import httpx
 import asyncio
@@ -8,101 +7,28 @@
 import traceback
 from pathlib import Path
 from bs4 import BeautifulSoup
-from contextlib import asynccontextmanager
 
-from httpx._models import Cookies
 from socksio.exceptions import SOCKSError
 
+from bbot.core.engine import EngineClient
 from bbot.errors import WordlistError, CurlError
 from bbot.core.helpers.ratelimiter import RateLimiter
 
 from bs4 import MarkupResemblesLocatorWarning
 from bs4.builder import XMLParsedAsHTMLWarning
 
+from .engine import HTTPEngine
+
 warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
 warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
 
 log = logging.getLogger("bbot.core.helpers.web")
 
 
-class DummyCookies(Cookies):
-    def extract_cookies(self, *args, **kwargs):
-        pass
-
-
-class BBOTAsyncClient(httpx.AsyncClient):
-    """
-    A subclass of httpx.AsyncClient tailored with BBOT-specific configurations and functionalities.
-    This class provides rate limiting, logging, configurable timeouts, user-agent customization, custom
-    headers, and proxy settings. Additionally, it allows the disabling of cookies, making it suitable
-    for use across an entire scan.
-
-    Attributes:
-        _bbot_scan (object): BBOT scan object containing configuration details.
-        _rate_limiter (RateLimiter): A rate limiter object to limit web requests.
-        _persist_cookies (bool): Flag to determine whether cookies should be persisted across requests.
-
-    Examples:
-        >>> async with BBOTAsyncClient(_bbot_scan=bbot_scan_object) as client:
-        >>>     response = await client.request("GET", "https://example.com")
-        >>>     print(response.status_code)
-        200
-    """
+class WebHelper(EngineClient):
 
-    def __init__(self, *args, **kwargs):
-        self._preset = kwargs.pop("_preset")
-        web_requests_per_second = self._preset.config.get("web_requests_per_second", 100)
-        self._rate_limiter = RateLimiter(web_requests_per_second, "Web")
-
-        http_debug = self._preset.config.get("http_debug", None)
-        if http_debug:
-            log.trace(f"Creating AsyncClient: {args}, {kwargs}")
-
-        self._persist_cookies = kwargs.pop("persist_cookies", True)
-
-        # timeout
-        http_timeout = self._preset.config.get("http_timeout", 20)
-        if not "timeout" in kwargs:
-            kwargs["timeout"] = http_timeout
-
-        # headers
-        headers = kwargs.get("headers", None)
-        if headers is None:
-            headers = {}
-        # user agent
-        user_agent = self._preset.config.get("user_agent", "BBOT")
-        if "User-Agent" not in headers:
-            headers["User-Agent"] = user_agent
-        kwargs["headers"] = headers
-        # proxy
-        proxies = self._preset.config.get("http_proxy", None)
-        kwargs["proxies"] = proxies
-
-        super().__init__(*args, **kwargs)
-        if not self._persist_cookies:
-            self._cookies = DummyCookies()
+    SERVER_CLASS = HTTPEngine
 
-    async def request(self, *args, **kwargs):
-        async with self._rate_limiter:
-            return await super().request(*args, **kwargs)
-
-    def build_request(self, *args, **kwargs):
-        request = super().build_request(*args, **kwargs)
-        # add custom headers if the URL is in-scope
-        if self._preset.in_scope(str(request.url)):
-            for hk, hv in self._preset.config.get("http_headers", {}).items():
-                # don't clobber headers
-                if hk not in request.headers:
-                    request.headers[hk] = hv
-        return request
-
-    def _merge_cookies(self, cookies):
-        if self._persist_cookies:
-            return super()._merge_cookies(cookies)
-        return cookies
-
-
-class WebHelper:
     """
     Main utility class for managing HTTP operations in BBOT. It serves as a wrapper around the BBOTAsyncClient,
     which itself is a subclass of httpx.AsyncClient. The class provides functionalities to make HTTP requests,
@@ -126,26 +52,10 @@ class WebHelper:
         >>> filename = await self.helpers.wordlist("https://www.evilcorp.com/wordlist.txt")
     """
 
-    client_only_options = (
-        "retries",
-        "max_redirects",
-    )
-
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
-        self.http_debug = self.parent_helper.config.get("http_debug", False)
-        self._ssl_context_noverify = None
-        self.ssl_verify = self.parent_helper.config.get("ssl_verify", False)
-        if self.ssl_verify is False:
-            self.ssl_verify = self.ssl_context_noverify()
-        self.web_client = self.AsyncClient(persist_cookies=False)
-
-    def AsyncClient(self, *args, **kwargs):
-        kwargs["_preset"] = self.parent_helper.preset
-        retries = kwargs.pop("retries", self.parent_helper.config.get("http_retries", 1))
-        kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=self.ssl_verify)
-        kwargs["verify"] = self.ssl_verify
-        return BBOTAsyncClient(*args, **kwargs)
+        self.config = self.parent_helper.config
+        super().__init__(server_kwargs={"config": self.config})
 
     async def request(self, *args, **kwargs):
         """
@@ -191,47 +101,8 @@ async def request(self, *args, **kwargs):
         Note:
             If the web request fails, it will return None unless `raise_error` is `True`.
         """
-
-        raise_error = kwargs.pop("raise_error", False)
-        # TODO: use this
-        cache_for = kwargs.pop("cache_for", None)  # noqa
-
-        client = kwargs.get("client", self.web_client)
-
-        # allow vs follow, httpx why??
-        allow_redirects = kwargs.pop("allow_redirects", None)
-        if allow_redirects is not None and "follow_redirects" not in kwargs:
-            kwargs["follow_redirects"] = allow_redirects
-
-        # in case of URL only, assume GET request
-        if len(args) == 1:
-            kwargs["url"] = args[0]
-            args = []
-
-        url = kwargs.get("url", "")
-
-        if not args and "method" not in kwargs:
-            kwargs["method"] = "GET"
-
-        client_kwargs = {}
-        for k in list(kwargs):
-            if k in self.client_only_options:
-                v = kwargs.pop(k)
-                client_kwargs[k] = v
-
-        if client_kwargs:
-            client = self.AsyncClient(**client_kwargs)
-
-        async with self._acatch(url, raise_error):
-            if self.http_debug:
-                logstr = f"Web request: {str(args)}, {str(kwargs)}"
-                log.trace(logstr)
-            response = await client.request(*args, **kwargs)
-            if self.http_debug:
-                log.trace(
-                    f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
-                )
-            return response
+        self.log.critical(f"CLIENT {args} / {kwargs}")
+        return await self.run_and_return("request", *args, **kwargs)
 
     async def download(self, url, **kwargs):
         """
@@ -629,77 +500,6 @@ def beautifulsoup(
             log.debug(f"Error parsing beautifulsoup: {e}")
             return False
 
-    def ssl_context_noverify(self):
-        if self._ssl_context_noverify is None:
-            ssl_context = ssl.create_default_context()
-            ssl_context.check_hostname = False
-            ssl_context.verify_mode = ssl.CERT_NONE
-            ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3
-            ssl_context.set_ciphers("ALL:@SECLEVEL=0")
-            ssl_context.options |= 0x4  # Add the OP_LEGACY_SERVER_CONNECT option
-            self._ssl_context_noverify = ssl_context
-        return self._ssl_context_noverify
-
-    @asynccontextmanager
-    async def _acatch(self, url, raise_error):
-        """
-        Asynchronous context manager to handle various httpx errors during a request.
-
-        Yields:
-            None
-
-        Note:
-            This function is internal and should generally not be used directly.
-            `url`, `args`, `kwargs`, and `raise_error` should be in the same context as this function.
-        """
-        try:
-            yield
-        except httpx.TimeoutException:
-            if raise_error:
-                raise
-            else:
-                log.verbose(f"HTTP timeout to URL: {url}")
-        except httpx.ConnectError:
-            if raise_error:
-                raise
-            else:
-                log.debug(f"HTTP connect failed to URL: {url}")
-        except httpx.HTTPError as e:
-            if raise_error:
-                raise
-            else:
-                log.trace(f"Error with request to URL: {url}: {e}")
-                log.trace(traceback.format_exc())
-        except ssl.SSLError as e:
-            msg = f"SSL error with request to URL: {url}: {e}"
-            if raise_error:
-                raise httpx.RequestError(msg)
-            else:
-                log.trace(msg)
-                log.trace(traceback.format_exc())
-        except anyio.EndOfStream as e:
-            msg = f"AnyIO error with request to URL: {url}: {e}"
-            if raise_error:
-                raise httpx.RequestError(msg)
-            else:
-                log.trace(msg)
-                log.trace(traceback.format_exc())
-        except SOCKSError as e:
-            msg = f"SOCKS error with request to URL: {url}: {e}"
-            if raise_error:
-                raise httpx.RequestError(msg)
-            else:
-                log.trace(msg)
-                log.trace(traceback.format_exc())
-        except BaseException as e:
-            # don't log if the error is the result of an intentional cancellation
-            if not any(
-                isinstance(_e, asyncio.exceptions.CancelledError) for _e in self.parent_helper.get_exception_chain(e)
-            ):
-                log.trace(f"Unhandled exception with request to URL: {url}: {e}")
-                log.trace(traceback.format_exc())
-            raise
-
 
 user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
 pass_keywords = [re.compile(r, re.I) for r in ["pass"]]
diff --git a/bbot/errors.py b/bbot/errors.py
index e50e581cd..153601813 100644
--- a/bbot/errors.py
+++ b/bbot/errors.py
@@ -72,3 +72,7 @@ class PresetConditionError(BBOTError):
 
 class PresetAbortError(PresetConditionError):
     pass
+
+
+class BBOTEngineError(BBOTError):
+    pass
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index 14da286d0..dc9116e0f 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -3,6 +3,13 @@
 from ..bbot_fixtures import *
 
 
+@pytest.mark.asyncio
+async def test_web_engine(bbot_scanner):
+    scan = bbot_scanner()
+    response = await scan.helpers.request("http://example.com")
+    log.critical(response)
+
+
 @pytest.mark.asyncio
 async def test_web_helpers(bbot_scanner, bbot_httpserver):
     scan1 = bbot_scanner("8.8.8.8")

From 27e9ae89136824ef6a44fe2b5365eb3f7a1e104e Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Thu, 2 May 2024 13:21:25 +0100
Subject: [PATCH 056/220] Remove de-duplicating in trufflehog module

---
 bbot/modules/trufflehog.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index bc0b2b230..0ec607457 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -85,7 +85,6 @@ async def execute_trufflehog(self, module, path):
             command.append("filesystem")
             command.append(path)
 
-        emitted_raw_results = set()
         stats_file = self.helpers.tempfile_tail(callback=self.log_trufflehog_status)
         try:
             with open(stats_file, "w") as stats_fh:
@@ -106,10 +105,6 @@ async def execute_trufflehog(self, module, path):
 
                     source_metadata = j.get("SourceMetadata", {})
 
-                    if raw_result in emitted_raw_results:
-                        continue
-
-                    emitted_raw_results.add(raw_result)
                     yield (decoder_name, detector_name, raw_result, verified, source_metadata)
         finally:
             stats_file.unlink()

From c2e09ae361f4de202744b1917c266c2594ed6294 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Thu, 2 May 2024 13:24:29 +0100
Subject: [PATCH 057/220] Change description for docker_pull

---
 bbot/modules/docker_pull.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py
index 4515e6d4a..86c7a07a7 100644
--- a/bbot/modules/docker_pull.py
+++ b/bbot/modules/docker_pull.py
@@ -50,7 +50,7 @@ async def handle_event(self, event):
         if repo_path:
             self.verbose(f"Downloaded docker repository {repo_url} to {repo_path}")
             codebase_event = self.make_event(
-                {"path": str(repo_path), "description": f"Docker image: {repo_url}"},
+                {"path": str(repo_path), "description": f"Docker image repository: {repo_url}"},
                 "FILESYSTEM",
                 tags=["docker", "tarball"],
                 source=event,

From 25e971c030cad206e06c254879f6cd4a73573835 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 3 May 2024 10:49:18 -0400
Subject: [PATCH 058/220] more WIP web engine

---
 bbot/core/helpers/web/engine.py   | 84 ++++++++++++++++++++++++++++---
 bbot/core/helpers/web/web.py      | 60 +++++-----------------
 bbot/test/test_step_1/test_web.py | 24 +++++++--
 3 files changed, 112 insertions(+), 56 deletions(-)

diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 9e30dbb8b..3a13bbb2d 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -1,9 +1,17 @@
 import ssl
+import anyio
 import httpx
+import asyncio
+import logging
+import traceback
 from httpx._models import Cookies
+from socksio.exceptions import SOCKSError
 from contextlib import asynccontextmanager
 
 from bbot.core.engine import EngineServer
+from bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain
+
+log = logging.getLogger("bbot.core.helpers.web.engine")
 
 
 class DummyCookies(Cookies):
@@ -31,7 +39,6 @@ class BBOTAsyncClient(httpx.AsyncClient):
 
     def __init__(self, *args, **kwargs):
         self._config = kwargs.pop("_config")
-        web_requests_per_second = self._config.get("web_requests_per_second", 100)
 
         http_debug = self._config.get("http_debug", None)
         if http_debug:
@@ -84,6 +91,7 @@ class HTTPEngine(EngineServer):
         0: "request",
         1: "request_batch",
         2: "request_custom_batch",
+        3: "download",
         99: "_mock",
     }
 
@@ -145,14 +153,80 @@ async def request(self, *args, **kwargs):
         async with self._acatch(url, raise_error):
             if self.http_debug:
                 logstr = f"Web request: {str(args)}, {str(kwargs)}"
-                log.trace(logstr)
+                self.log.trace(logstr)
             response = await client.request(*args, **kwargs)
             if self.http_debug:
-                log.trace(
+                self.log.trace(
                     f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
                 )
             return response
 
+    async def request_batch(self, urls, *args, threads=10, **kwargs):
+        tasks = {}
+
+        def new_task(url):
+            task = asyncio.create_task(self.request(url, *args, **kwargs))
+            tasks[task] = url
+
+        urls = list(urls)
+        for _ in range(threads):  # Start initial batch of tasks
+            if urls:  # Ensure there are args to process
+                new_task(urls.pop(0))
+
+        while tasks:  # While there are tasks pending
+            # Wait for the first task to complete
+            done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+
+            for task in done:
+                results = task.result()
+                url = tasks.pop(task)
+
+                if results:
+                    yield (url, results)
+
+                if urls:  # Start a new task for each one completed, if URLs remain
+                    new_task(urls.pop(0))
+
+    async def download(self, url, **kwargs):
+        follow_redirects = kwargs.pop("follow_redirects", True)
+        filename = kwargs.pop("filename")
+        max_size = kwargs.pop("max_size", None)
+        warn = kwargs.pop("warn", True)
+        raise_error = kwargs.pop("raise_error", False)
+        if max_size is not None:
+            max_size = human_to_bytes(max_size)
+        kwargs["follow_redirects"] = follow_redirects
+        if not "method" in kwargs:
+            kwargs["method"] = "GET"
+        try:
+            total_size = 0
+            chunk_size = 8192
+
+            async with self._acatch(url, raise_error=True), self.web_client.stream(url=url, **kwargs) as response:
+                status_code = getattr(response, "status_code", 0)
+                self.log.debug(f"Download result: HTTP {status_code}")
+                if status_code != 0:
+                    response.raise_for_status()
+                    with open(filename, "wb") as f:
+                        agen = response.aiter_bytes(chunk_size=chunk_size)
+                        async for chunk in agen:
+                            if max_size is not None and total_size + chunk_size > max_size:
+                                self.log.verbose(
+                                    f"Filesize of {url} exceeds {bytes_to_human(max_size)}, file will be truncated"
+                                )
+                                agen.aclose()
+                                break
+                            total_size += chunk_size
+                            f.write(chunk)
+                    return True
+        except httpx.HTTPError as e:
+            log_fn = self.log.verbose
+            if warn:
+                log_fn = self.log.warning
+            log_fn(f"Failed to download {url}: {e}")
+            if raise_error:
+                raise
+
     def ssl_context_noverify(self):
         if self._ssl_context_noverify is None:
             ssl_context = ssl.create_default_context()
@@ -217,9 +291,7 @@ async def _acatch(self, url, raise_error):
                 log.trace(traceback.format_exc())
         except BaseException as e:
             # don't log if the error is the result of an intentional cancellation
-            if not any(
-                isinstance(_e, asyncio.exceptions.CancelledError) for _e in self.parent_helper.get_exception_chain(e)
-            ):
+            if not any(isinstance(_e, asyncio.exceptions.CancelledError) for _e in get_exception_chain(e)):
                 log.trace(f"Unhandled exception with request to URL: {url}: {e}")
                 log.trace(traceback.format_exc())
             raise
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 6d44cca61..09bc3b581 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -1,18 +1,12 @@
 import re
-import anyio
-import httpx
-import asyncio
 import logging
 import warnings
 import traceback
 from pathlib import Path
 from bs4 import BeautifulSoup
 
-from socksio.exceptions import SOCKSError
-
 from bbot.core.engine import EngineClient
 from bbot.errors import WordlistError, CurlError
-from bbot.core.helpers.ratelimiter import RateLimiter
 
 from bs4 import MarkupResemblesLocatorWarning
 from bs4.builder import XMLParsedAsHTMLWarning
@@ -101,9 +95,16 @@ async def request(self, *args, **kwargs):
         Note:
             If the web request fails, it will return None unless `raise_error` is `True`.
         """
-        self.log.critical(f"CLIENT {args} / {kwargs}")
         return await self.run_and_return("request", *args, **kwargs)
 
+    async def request_batch(self, urls, *args, **kwargs):
+        async for _ in self.run_and_yield("request_batch", urls, *args, **kwargs):
+            yield _
+
+    async def request_custom_batch(self, urls_and_args):
+        async for _ in self.run_and_yield("request_custom_batch", urls_and_args):
+            yield _
+
     async def download(self, url, **kwargs):
         """
         Asynchronous function for downloading files from a given URL. Supports caching with an optional
@@ -129,56 +130,21 @@ async def download(self, url, **kwargs):
         """
         success = False
         filename = kwargs.pop("filename", self.parent_helper.cache_filename(url))
-        follow_redirects = kwargs.pop("follow_redirects", True)
+        filename = Path(filename).resolve()
+        kwargs["filename"] = filename
         max_size = kwargs.pop("max_size", None)
-        warn = kwargs.pop("warn", True)
-        raise_error = kwargs.pop("raise_error", False)
         if max_size is not None:
             max_size = self.parent_helper.human_to_bytes(max_size)
+            kwargs["max_size"] = max_size
         cache_hrs = float(kwargs.pop("cache_hrs", -1))
-        total_size = 0
-        chunk_size = 8192
-        log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}")
         if cache_hrs > 0 and self.parent_helper.is_cached(url):
             log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}")
             success = True
         else:
-            # kwargs["raise_error"] = True
-            # kwargs["stream"] = True
-            kwargs["follow_redirects"] = follow_redirects
-            if not "method" in kwargs:
-                kwargs["method"] = "GET"
-            try:
-                async with self._acatch(url, raise_error=True), self.AsyncClient().stream(
-                    url=url, **kwargs
-                ) as response:
-                    status_code = getattr(response, "status_code", 0)
-                    log.debug(f"Download result: HTTP {status_code}")
-                    if status_code != 0:
-                        response.raise_for_status()
-                        with open(filename, "wb") as f:
-                            agen = response.aiter_bytes(chunk_size=chunk_size)
-                            async for chunk in agen:
-                                if max_size is not None and total_size + chunk_size > max_size:
-                                    log.verbose(
-                                        f"Filesize of {url} exceeds {self.parent_helper.bytes_to_human(max_size)}, file will be truncated"
-                                    )
-                                    agen.aclose()
-                                    break
-                                total_size += chunk_size
-                                f.write(chunk)
-                        success = True
-            except httpx.HTTPError as e:
-                log_fn = log.verbose
-                if warn:
-                    log_fn = log.warning
-                log_fn(f"Failed to download {url}: {e}")
-                if raise_error:
-                    raise
-                return
+            success = await self.run_and_return("download", url, **kwargs)
 
         if success:
-            return filename.resolve()
+            return filename
 
     async def wordlist(self, path, lines=None, **kwargs):
         """
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index dc9116e0f..aeac2ba2f 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -4,10 +4,28 @@
 
 
 @pytest.mark.asyncio
-async def test_web_engine(bbot_scanner):
+async def test_web_engine(bbot_scanner, bbot_httpserver):
+
+    url = bbot_httpserver.url_for("/test")
+    bbot_httpserver.expect_request(uri="/test").respond_with_data("hello_there")
+
     scan = bbot_scanner()
-    response = await scan.helpers.request("http://example.com")
-    log.critical(response)
+
+    # request
+    response = await scan.helpers.request(url)
+    assert response.status_code > 0
+    assert response.text == "hello_there"
+
+    # request_batch
+    responses = [r async for r in scan.helpers.request_batch([url] * 100)]
+    assert len(responses) == 100
+    assert all([r[0] == url for r in responses])
+    assert all([r[1].status_code > 0 and r[1].text == "hello_there" for r in responses])
+
+    # download
+    filename = await scan.helpers.download(url)
+    file_content = open(filename).read()
+    assert file_content == "hello_there"
 
 
 @pytest.mark.asyncio

From 4c6362064f2e7bcec8762f1443b1f5569e162781 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Fri, 3 May 2024 18:21:25 +0100
Subject: [PATCH 059/220] emit the full event logs as `FILESYSTEM` events to be
 scanned to avoid duplication

---
 bbot/modules/github_workflows.py              | 25 +++++++++++++------
 .../test_module_github_workflows.py           | 24 ++++++------------
 2 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 8999c41a3..03d8e55f0 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -1,3 +1,6 @@
+import zipfile
+import fnmatch
+
 from bbot.modules.templates.github import github
 
 
@@ -40,16 +43,14 @@ async def handle_event(self, event):
             for run in await self.get_workflow_runs(owner, repo, workflow_id):
                 run_id = run.get("id")
                 self.log.debug(f"Downloading logs for {workflow_name}/{run_id} in {owner}/{repo}")
-                log_path = await self.download_run_logs(owner, repo, run_id)
-                if log_path:
-                    self.verbose(f"Downloaded repository workflow logs to {log_path}")
+                for log in await self.download_run_logs(owner, repo, run_id):
                     logfile_event = self.make_event(
                         {
-                            "path": str(log_path),
+                            "path": str(log),
                             "description": f"Workflow run logs from https://github.com/{owner}/{repo}/actions/runs/{run_id}",
                         },
                         "FILESYSTEM",
-                        tags=["zipfile"],
+                        tags=["textfile"],
                         source=event,
                     )
                     logfile_event.scope_distance = event.scope_distance
@@ -119,8 +120,8 @@ async def download_run_logs(self, owner, repo, run_id):
                 warn=False,
             )
             self.info(f"Downloaded logs for {owner}/{repo}/{run_id} to {file_destination}")
-            return file_destination
         except Exception as e:
+            file_destination = None
             response = getattr(e, "response", None)
             status_code = getattr(response, "status_code", 0)
             if status_code == 403:
@@ -131,4 +132,14 @@ async def download_run_logs(self, owner, repo, run_id):
                 self.info(
                     f"The logs for {owner}/{repo}/{run_id} have expired and are no longer available (status: {status_code})"
                 )
-        return None
+        # Secrets are duplicated in the individual workflow steps so just extract the main log files from the top folder
+        if file_destination:
+            main_logs = []
+            with zipfile.ZipFile(file_destination, "r") as logzip:
+                for name in logzip.namelist():
+                    if fnmatch.fnmatch(name, "*.txt") and not "/" in name:
+                        logzip.extract(name, folder)
+                        main_logs.append(folder / name)
+            return main_logs
+        else:
+            return []
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
index 76bd55814..c5bdf6d07 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -12,6 +12,8 @@ class TestGithub_Workflows(ModuleTestBase):
     data = io.BytesIO()
     with zipfile.ZipFile(data, mode="w", compression=zipfile.ZIP_DEFLATED) as zipfile:
         zipfile.writestr("test.txt", "This is some test data")
+        zipfile.writestr("test2.txt", "This is some more test data")
+        zipfile.writestr("folder/test3.txt", "This is yet more test data")
     data.seek(0)
     zip_content = data.getvalue()
 
@@ -437,7 +439,7 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-        assert len(events) == 6
+        assert len(events) == 7
         assert 1 == len(
             [
                 e
@@ -469,18 +471,8 @@ def check(self, module_test, events):
                 and e.scope_distance == 1
             ]
         ), "Failed to find blacklanternsecurity github repo"
-        filesystem_events = [
-            e
-            for e in events
-            if e.type == "FILESYSTEM"
-            and "workflow_logs/blacklanternsecurity/bbot/run_8839360698.zip" in e.data["path"]
-            and "zipfile" in e.tags
-            and e.scope_distance == 1
-        ]
-        assert 1 == len(filesystem_events), "Failed to download workflow logs"
-        filesystem_event = filesystem_events[0]
-        file = Path(filesystem_event.data["path"])
-        assert file.is_file(), "Destination zip does not exist"
-        with zipfile.ZipFile(file, "r") as zip_ref:
-            assert "test.txt" in zip_ref.namelist(), "test.txt not in zip"
-            assert zip_ref.read("test.txt") == b"This is some test data", "test.txt contents incorrect"
+        filesystem_events = [e for e in events if e.type == "FILESYSTEM"]
+        assert 2 == len(filesystem_events), filesystem_events
+        for filesystem_event in filesystem_events:
+            file = Path(filesystem_event.data["path"])
+            assert file.is_file(), "Destination file does not exist"

From 722e961e4cc2af279457c14bd30b1e64ea83a2dc Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 3 May 2024 17:06:57 -0400
Subject: [PATCH 060/220] remove unneeded filter

---
 bbot/modules/base.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 92e341e18..15667a73b 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -730,13 +730,6 @@ async def _event_postcheck_inner(self, event):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
-        # don't send out-of-scope targets to active modules (excluding portscanners, because they can handle it)
-        # this only takes effect if your target and whitelist are different
-        # TODO: the logic here seems incomplete, it could probably use some work.
-        if "active" in self.flags and "portscan" not in self.flags:
-            if "target" in event.tags and event not in self.scan.whitelist:
-                return False, "it is not in whitelist and module has active flag"
-
         # check scope distance
         filter_result, reason = self._scope_distance_check(event)
         if not filter_result:

From ace01655f8254d517c23d5292106012e86cddde6 Mon Sep 17 00:00:00 2001
From: BBOT Docs Autopublish <info@blacklanternsecurity.com>
Date: Fri, 3 May 2024 21:26:19 +0000
Subject: [PATCH 061/220] Refresh module docs

---
 README.md                       | 4 ++--
 docs/modules/list_of_modules.md | 2 +-
 docs/scanning/index.md          | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 885d740ca..f97980608 100644
--- a/README.md
+++ b/README.md
@@ -267,9 +267,9 @@ For a full list of modules, including the data types consumed and emitted by eac
 | Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 |------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | safe             | 80          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
-| passive          | 59          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                 |
+| passive          | 60          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
 | subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                           |
-| active           | 43          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dockerhub, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                 |
+| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                            |
 | web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md
index bd3e58997..190b567e9 100644
--- a/docs/modules/list_of_modules.md
+++ b/docs/modules/list_of_modules.md
@@ -14,7 +14,6 @@
 | bucket_google        | scan     | No              | Check for Google object storage related to target                                                                                       | active, cloud-enum, safe, web-basic, web-thorough                              | DNS_NAME, STORAGE_BUCKET                                                                                                                | FINDING, STORAGE_BUCKET                                     |
 | bypass403            | scan     | No              | Check 403 pages for common bypasses                                                                                                     | active, aggressive, web-thorough                                               | URL                                                                                                                                     | FINDING                                                     |
 | dastardly            | scan     | No              | Lightweight web application security scanner                                                                                            | active, aggressive, deadly, slow, web-thorough                                 | HTTP_RESPONSE                                                                                                                           | FINDING, VULNERABILITY                                      |
-| dockerhub            | scan     | No              | Search for docker repositories of discovered orgs/usernames                                                                             | active, safe                                                                   | ORG_STUB, SOCIAL                                                                                                                        | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED                     |
 | dotnetnuke           | scan     | No              | Scan for critical DotNetNuke (DNN) vulnerabilities                                                                                      | active, aggressive, web-thorough                                               | HTTP_RESPONSE                                                                                                                           | TECHNOLOGY, VULNERABILITY                                   |
 | ffuf                 | scan     | No              | A fast web fuzzer written in Go                                                                                                         | active, aggressive, deadly                                                     | URL                                                                                                                                     | URL_UNVERIFIED                                              |
 | ffuf_shortnames      | scan     | No              | Use ffuf in combination IIS shortnames                                                                                                  | active, aggressive, iis-shortnames, web-thorough                               | URL_HINT                                                                                                                                | URL_UNVERIFIED                                              |
@@ -68,6 +67,7 @@
 | dnscommonsrv         | scan     | No              | Check for common SRV records                                                                                                            | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | dnsdumpster          | scan     | No              | Query dnsdumpster for subdomains                                                                                                        | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | docker_pull          | scan     | No              | Download images from a docker repository                                                                                                | passive, safe, slow                                                            | CODE_REPOSITORY                                                                                                                         | FILESYSTEM                                                  |
+| dockerhub            | scan     | No              | Search for docker repositories of discovered orgs/usernames                                                                             | passive, safe                                                                  | ORG_STUB, SOCIAL                                                                                                                        | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED                     |
 | emailformat          | scan     | No              | Query email-format.com for email addresses                                                                                              | email-enum, passive, safe                                                      | DNS_NAME                                                                                                                                | EMAIL_ADDRESS                                               |
 | fullhunt             | scan     | Yes             | Query the fullhunt.io API for subdomains                                                                                                | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | git_clone            | scan     | No              | Clone code github repositories                                                                                                          | passive, safe, slow                                                            | CODE_REPOSITORY                                                                                                                         | FILESYSTEM                                                  |
diff --git a/docs/scanning/index.md b/docs/scanning/index.md
index 4d522a309..0cf499891 100644
--- a/docs/scanning/index.md
+++ b/docs/scanning/index.md
@@ -110,9 +110,9 @@ A single module can have multiple flags. For example, the `securitytrails` modul
 | Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 |------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | safe             | 80          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
-| passive          | 59          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                 |
+| passive          | 60          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
 | subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                           |
-| active           | 43          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dockerhub, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                 |
+| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                            |
 | web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |

From 76efe687db71498431b83bfd051f1fbd45f40929 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 15:13:43 -0400
Subject: [PATCH 062/220] Ensure bbot config

---
 bbot/core/config/files.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py
index 6547d02ec..0f05c0b50 100644
--- a/bbot/core/config/files.py
+++ b/bbot/core/config/files.py
@@ -49,6 +49,7 @@ def _get_config(self, filename, name="config"):
             return OmegaConf.create()
 
     def get_custom_config(self):
+        self.ensure_config_file()
         return self._get_config(self.config_filename, name="config")
 
     def get_default_config(self):

From 947c2d52dbe866daa9dcf8bc2c36fe8d5a3372fa Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 15:38:01 -0400
Subject: [PATCH 063/220] small boost in import speed

---
 bbot/core/config/files.py |  1 +
 bbot/core/core.py         |  7 +++----
 bbot/scanner/__init__.py  |  1 -
 bbot/scanner/scanner.py   | 10 ++++++----
 4 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py
index 0f05c0b50..7bf568015 100644
--- a/bbot/core/config/files.py
+++ b/bbot/core/config/files.py
@@ -49,6 +49,7 @@ def _get_config(self, filename, name="config"):
             return OmegaConf.create()
 
     def get_custom_config(self):
+        assert False
         self.ensure_config_file()
         return self._get_config(self.config_filename, name="config")
 
diff --git a/bbot/core/core.py b/bbot/core/core.py
index 1c43e5035..d808f810f 100644
--- a/bbot/core/core.py
+++ b/bbot/core/core.py
@@ -52,10 +52,6 @@ def __init__(self):
         self._config = None
         self._custom_config = None
 
-        # ensure bbot home dir
-        if not "home" in self.config:
-            self.custom_config["home"] = "~/.bbot"
-
         # bare minimum == logging
         self.logger
         self.log = logging.getLogger("bbot.core")
@@ -105,6 +101,9 @@ def default_config(self):
         global DEFAULT_CONFIG
         if DEFAULT_CONFIG is None:
             self.default_config = self.files_config.get_default_config()
+            # ensure bbot home dir
+            if not "home" in self.default_config:
+                self.default_config["home"] = "~/.bbot"
         return DEFAULT_CONFIG
 
     @default_config.setter
diff --git a/bbot/scanner/__init__.py b/bbot/scanner/__init__.py
index 1622f4c20..cc993af8a 100644
--- a/bbot/scanner/__init__.py
+++ b/bbot/scanner/__init__.py
@@ -1,2 +1 @@
-from .preset import Preset
 from .scanner import Scanner
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 06f55c340..69b26c30f 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -11,10 +11,6 @@
 
 from bbot import __version__
 
-
-from .preset import Preset
-from .stats import ScanStats
-from .dispatcher import Dispatcher
 from bbot.core.event import make_event
 from .manager import ScanIngress, ScanEgress
 from bbot.core.helpers.misc import sha1, rand_string
@@ -126,6 +122,8 @@ def __init__(
         preset = kwargs.pop("preset", None)
         kwargs["_log"] = True
         if preset is None:
+            from .preset import Preset
+
             preset = Preset(*targets, **kwargs)
         else:
             if not isinstance(preset, Preset):
@@ -168,11 +166,15 @@ def __init__(
         self.dummy_modules = {}
 
         if dispatcher is None:
+            from .dispatcher import Dispatcher
+
             self.dispatcher = Dispatcher()
         else:
             self.dispatcher = dispatcher
         self.dispatcher.set_scan(self)
 
+        from .stats import ScanStats
+
         self.stats = ScanStats(self)
 
         # scope distance

From ad2e3286a6f57837a8f3a1014e011fd01f056eef Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 15:38:57 -0400
Subject: [PATCH 064/220] remove assertion

---
 bbot/core/config/files.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py
index 7bf568015..0f05c0b50 100644
--- a/bbot/core/config/files.py
+++ b/bbot/core/config/files.py
@@ -49,7 +49,6 @@ def _get_config(self, filename, name="config"):
             return OmegaConf.create()
 
     def get_custom_config(self):
-        assert False
         self.ensure_config_file()
         return self._get_config(self.config_filename, name="config")
 

From 8912aa22f169ceedf40a99633b7567826d7d0232 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 15:42:15 -0400
Subject: [PATCH 065/220] fix import error

---
 bbot/scanner/scanner.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 69b26c30f..f7c22a859 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -121,8 +121,9 @@ def __init__(
 
         preset = kwargs.pop("preset", None)
         kwargs["_log"] = True
+        from .preset import Preset
+
         if preset is None:
-            from .preset import Preset
 
             preset = Preset(*targets, **kwargs)
         else:

From 3e63198008b1ecba08bbb8e9e16ab895bf9f3815 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 15:42:32 -0400
Subject: [PATCH 066/220] formatting

---
 bbot/scanner/scanner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index f7c22a859..541c2d7a9 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -121,10 +121,10 @@ def __init__(
 
         preset = kwargs.pop("preset", None)
         kwargs["_log"] = True
+
         from .preset import Preset
 
         if preset is None:
-
             preset = Preset(*targets, **kwargs)
         else:
             if not isinstance(preset, Preset):

From 4d8e7cdd9743b687070bbc04482a9af2313ba1a9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 16:21:31 -0400
Subject: [PATCH 067/220] improve import friendliness

---
 bbot/__init__.py                             | 2 ++
 bbot/scanner/__init__.py                     | 1 +
 bbot/scripts/docs.py                         | 2 +-
 bbot/test/test_step_1/test__module__tests.py | 2 +-
 bbot/test/test_step_1/test_python_api.py     | 2 +-
 5 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/bbot/__init__.py b/bbot/__init__.py
index 8e016095f..8746d8131 100644
--- a/bbot/__init__.py
+++ b/bbot/__init__.py
@@ -1,2 +1,4 @@
 # version placeholder (replaced by poetry-dynamic-versioning)
 __version__ = "v0.0.0"
+
+from .scanner import Scanner, Preset
diff --git a/bbot/scanner/__init__.py b/bbot/scanner/__init__.py
index cc993af8a..1622f4c20 100644
--- a/bbot/scanner/__init__.py
+++ b/bbot/scanner/__init__.py
@@ -1 +1,2 @@
+from .preset import Preset
 from .scanner import Scanner
diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py
index f8a5050a3..98bd48a2b 100755
--- a/bbot/scripts/docs.py
+++ b/bbot/scripts/docs.py
@@ -5,7 +5,7 @@
 import yaml
 from pathlib import Path
 
-from bbot.scanner import Preset
+from bbot import Preset
 
 DEFAULT_PRESET = Preset()
 
diff --git a/bbot/test/test_step_1/test__module__tests.py b/bbot/test/test_step_1/test__module__tests.py
index 9d88b1bcc..791e58f58 100644
--- a/bbot/test/test_step_1/test__module__tests.py
+++ b/bbot/test/test_step_1/test__module__tests.py
@@ -2,7 +2,7 @@
 import importlib
 from pathlib import Path
 
-from bbot.scanner import Preset
+from bbot import Preset
 from ..test_step_2.module_tests.base import ModuleTestBase
 
 log = logging.getLogger("bbot.test.modules")
diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py
index 19575e3ed..678593ed1 100644
--- a/bbot/test/test_step_1/test_python_api.py
+++ b/bbot/test/test_step_1/test_python_api.py
@@ -3,7 +3,7 @@
 
 @pytest.mark.asyncio
 async def test_python_api():
-    from bbot.scanner import Scanner
+    from bbot import Scanner
 
     # make sure events are properly yielded
     scan1 = Scanner("127.0.0.1")

From 63afacbebc110db702a7ad9652c2ad26d7362d23 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 19:34:49 -0400
Subject: [PATCH 068/220] fix github_org tests

---
 .../module_tests/test_module_github_org.py       | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py
index 5d999a721..4a368d908 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_org.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py
@@ -399,7 +399,7 @@ class TestGithub_Org_Custom_Target(TestGithub_Org):
     config_overrides = {"scope_report_distance": 10, "omit_event_types": [], "speculate": True}
 
     def check(self, module_test, events):
-        assert len(events) == 7
+        assert len(events) == 8
         assert 1 == len(
             [e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 1]
         )
@@ -411,6 +411,20 @@ def check(self, module_test, events):
                 and e.data["platform"] == "github"
                 and e.data["profile_name"] == "blacklanternsecurity"
                 and e.scope_distance == 1
+                and str(e.module) == "social"
+                and e.source.type == "URL_UNVERIFIED"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "github"
+                and e.data["profile_name"] == "blacklanternsecurity"
+                and e.scope_distance == 1
+                and str(e.module) == "github_org"
+                and e.source.type == "ORG_STUB"
             ]
         )
         assert 1 == len(

From 47161e5d5a225c6edce5c5bfd1cb1cba647a5a32 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 20:22:48 -0400
Subject: [PATCH 069/220] fix cli tests

---
 bbot/test/test_step_1/test_cli.py | 160 +++++++++++++++---------------
 1 file changed, 80 insertions(+), 80 deletions(-)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 524557fac..63eb8334b 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -65,148 +65,148 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert len(out.splitlines()) == 1
     assert out.count(".") > 1
 
+    # list modules
+    monkeypatch.setattr("sys.argv", ["bbot", "--list-modules"])
+    result = await cli._main()
+    assert result == None
+    out, err = capsys.readouterr()
     # internal modules
-    assert "| excavate" in caplog.text
+    assert "| excavate" in out
     # output modules
-    assert "| csv" in caplog.text
+    assert "| csv" in out
     # scan modules
-    assert "| wayback" in caplog.text
+    assert "| wayback" in out
+
+    # output dir and scan name
+    output_dir = bbot_test_dir / "bbot_cli_args_output"
+    scan_name = "bbot_cli_args_scan_name"
+    scan_dir = output_dir / scan_name
+    assert not output_dir.exists()
+    monkeypatch.setattr("sys.argv", ["bbot", "-o", str(output_dir), "-n", scan_name, "-y"])
+    result = await cli._main()
+    assert result == True
+    assert output_dir.is_dir()
+    assert scan_dir.is_dir()
+    assert "[SCAN]" in open(scan_dir / "output.txt").read()
+    assert "[INFO]" in open(scan_dir / "scan.log").read()
 
     # list module options
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "--list-module-options"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| modules.wayback.urls" in caplog.text
-    assert "| bool" in caplog.text
-    assert "| emit URLs in addition to DNS_NAMEs" in caplog.text
-    assert "| False" in caplog.text
-    assert "| modules.dnsbrute.wordlist" in caplog.text
-    assert "| modules.robots.include_allow" in caplog.text
+    assert "| modules.wayback.urls" in out
+    assert "| bool" in out
+    assert "| emit URLs in addition to DNS_NAMEs" in out
+    assert "| False" in out
+    assert "| modules.dnsbrute.wordlist" in out
+    assert "| modules.robots.include_allow" in out
 
     # list module options by flag
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "--list-module-options"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| modules.wayback.urls" in caplog.text
-    assert "| bool" in caplog.text
-    assert "| emit URLs in addition to DNS_NAMEs" in caplog.text
-    assert "| False" in caplog.text
-    assert "| modules.dnsbrute.wordlist" in caplog.text
-    assert not "| modules.robots.include_allow" in caplog.text
+    assert "| modules.wayback.urls" in out
+    assert "| bool" in out
+    assert "| emit URLs in addition to DNS_NAMEs" in out
+    assert "| False" in out
+    assert "| modules.dnsbrute.wordlist" in out
+    assert not "| modules.robots.include_allow" in out
 
     # list module options by module
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-m", "dnsbrute", "-lmo"])
     result = await cli._main()
+    out, err = capsys.readouterr()
+    assert result == None
+    assert out.count("modules.") == out.count("modules.dnsbrute.")
+    assert not "| modules.wayback.urls" in out
+    assert "| modules.dnsbrute.wordlist" in out
+    assert not "| modules.robots.include_allow" in out
+
+    # list output module options by module
+    monkeypatch.setattr("sys.argv", ["bbot", "-om", "stdout", "-lmo"])
+    result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert not "| modules.wayback.urls" in caplog.text
-    assert "| modules.dnsbrute.wordlist" in caplog.text
-    assert not "| modules.robots.include_allow" in caplog.text
+    assert out.count("modules.") == out.count("modules.stdout.")
 
     # list flags
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "--list-flags"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| safe" in caplog.text
-    assert "| Non-intrusive, safe to run" in caplog.text
-    assert "| active" in caplog.text
-    assert "| passive" in caplog.text
+    assert "| safe" in out
+    assert "| Non-intrusive, safe to run" in out
+    assert "| active" in out
+    assert "| passive" in out
 
     # list only a single flag
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "--list-flags"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert not "| safe" in caplog.text
-    assert "| active" in caplog.text
-    assert not "| passive" in caplog.text
+    assert not "| safe" in out
+    assert "| active" in out
+    assert not "| passive" in out
 
     # list multiple flags
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "active", "safe", "--list-flags"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| safe" in caplog.text
-    assert "| active" in caplog.text
-    assert not "| passive" in caplog.text
-
-    # custom target type
-    caplog.clear()
-    assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-t", "ORG:evilcorp"])
-    result = await cli._main()
-    assert result == True
-    assert "[ORG_STUB]          	evilcorp	TARGET" in caplog.text
-
-    # activate modules by flag
-    caplog.clear()
-    assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-f", "passive"])
-    result = await cli._main()
-    assert result == True
+    assert "| safe" in out
+    assert "| active" in out
+    assert not "| passive" in out
 
     # no args
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "Target:\n  -t TARGET [TARGET ...]" in caplog.text
+    assert "Target:\n  -t TARGET [TARGET ...]" in out
 
     # list modules
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-l"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| dnsbrute" in caplog.text
-    assert "| httpx" in caplog.text
-    assert "| robots" in caplog.text
+    assert "| dnsbrute " in out
+    assert "| httpx" in out
+    assert "| robots" in out
 
     # list modules by flag
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-l"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| dnsbrute" in caplog.text
-    assert "| httpx" in caplog.text
-    assert not "| robots" in caplog.text
+    assert "| dnsbrute " in out
+    assert "| httpx" in out
+    assert not "| robots" in out
 
     # list modules by flag + required flag
-    caplog.clear()
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-rf", "passive", "-l"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| dnsbrute" in caplog.text
-    assert not "| httpx" in caplog.text
+    assert "| dnsbrute " in out
+    assert not "| httpx" in out
 
     # list modules by flag + excluded flag
-    caplog.clear()
-    assert not caplog.text
     monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-ef", "active", "-l"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert "| dnsbrute" in caplog.text
-    assert not "| httpx" in caplog.text
+    assert "| dnsbrute " in out
+    assert not "| httpx" in out
 
     # list modules by flag + excluded module
-    caplog.clear()
-    assert not caplog.text
-    monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "dnsbrute", "dnsbrute_mutations", "-l"])
+    monkeypatch.setattr("sys.argv", ["bbot", "-f", "subdomain-enum", "-em", "dnsbrute", "-l"])
     result = await cli._main()
+    out, err = capsys.readouterr()
     assert result == None
-    assert not "| dnsbrute" in caplog.text
-    assert "| httpx" in caplog.text
+    assert not "| dnsbrute " in out
+    assert "| httpx" in out
 
     # output modules override
     caplog.clear()

From 07a061c4fb8f1015af0601f337f6e4d340e39a29 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 20:30:06 -0400
Subject: [PATCH 070/220] restore --install-all-deps test

---
 bbot/test/test_step_1/test_cli.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index b658f36bc..0d36408c0 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -309,9 +309,9 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert result == True, "-m nuclei failed to run with --allow-deadly"
 
     # install all deps
-    # monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"])
-    # success = await cli._main()
-    # assert success, "--install-all-deps failed for at least one module"
+    monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"])
+    success = await cli._main()
+    assert success == True, "--install-all-deps failed for at least one module"
 
 
 def test_cli_config_validation(monkeypatch, caplog):

From 31fb52bb45ba43fde955d323f7cb18010e3a30ce Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 4 May 2024 21:45:27 -0400
Subject: [PATCH 071/220] fix dnsbrute tests

---
 bbot/test/test_step_2/module_tests/test_module_dnsbrute.py   | 5 ++++-
 .../module_tests/test_module_dnsbrute_mutations.py           | 1 +
 .../test_step_2/module_tests/test_module_dnscommonsrv.py     | 4 +++-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
index bdbd2f6cb..d1c5e5cc9 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
@@ -12,7 +12,9 @@ async def setup_after_prep(self, module_test):
 
         async def new_run_live(*command, check=False, text=True, **kwargs):
             if "massdns" in command[:2]:
-                yield """{"name": "asdf.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "asdf.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
+                _input = [l async for l in kwargs["input"]]
+                if "asdf.blacklanternsecurity.com" in _input:
+                    yield """{"name": "asdf.blacklanternsecurity.com.", "type": "A", "class": "IN", "status": "NOERROR", "rx_ts": 1713974911725326170, "data": {"answers": [{"ttl": 86400, "type": "A", "class": "IN", "name": "asdf.blacklanternsecurity.com.", "data": "1.2.3.4."}]}, "flags": ["rd", "ra"], "resolver": "195.226.187.130:53", "proto": "UDP"}"""
             else:
                 async for _ in old_run_live(*command, check=False, text=True, **kwargs):
                     yield _
@@ -21,6 +23,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
 
         await module_test.mock_dns(
             {
+                "blacklanternsecurity.com": {"A": ["4.3.2.1"]},
                 "asdf.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
             }
         )
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py
index 2a56b2b65..0a7627f25 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute_mutations.py
@@ -31,6 +31,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
 
         await module_test.mock_dns(
             {
+                "blacklanternsecurity.com": {"A": ["1.2.3.4"]},
                 # targets
                 "rrrr.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
                 "asdff-ffdsa.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
index 3d3d670e1..8d54f4e3a 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
@@ -10,7 +10,9 @@ async def setup_after_prep(self, module_test):
 
         async def new_run_live(*command, check=False, text=True, **kwargs):
             if "massdns" in command[:2]:
-                yield """{"name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","type":"SRV","class":"IN","status":"NOERROR","rx_ts":1713974911725326170,"data":{"answers":[{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."},{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."}]},"flags":["rd","ra"],"resolver":"195.226.187.130:53","proto":"UDP"}"""
+                _input = [l async for l in kwargs["input"]]
+                if "_ldap._tcp.gc._msdcs.blacklanternsecurity.com" in _input:
+                    yield """{"name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","type":"SRV","class":"IN","status":"NOERROR","rx_ts":1713974911725326170,"data":{"answers":[{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."},{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."}]},"flags":["rd","ra"],"resolver":"195.226.187.130:53","proto":"UDP"}"""
             else:
                 async for _ in old_run_live(*command, check=False, text=True, **kwargs):
                     yield _

From 00759bc4a8adf74315263b21f76346d3d4a7dd99 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 5 May 2024 10:19:12 -0400
Subject: [PATCH 072/220] better pickle/unpickle error handling

---
 bbot/core/engine.py | 63 ++++++++++++++++++++++++++++++++++-----------
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 24781ab3b..269e7c645 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -16,18 +16,53 @@
 CMD_EXIT = 1000
 
 
-class EngineClient:
+error_sentinel = object()
+
+
+class EngineBase:
+    def __init__(self):
+        self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}")
+
+    def pickle(self, obj):
+        try:
+            return pickle.dumps(obj)
+        except Exception as e:
+            self.log.error(f"Error serializing object: {obj}: {e}")
+            self.log.trace(traceback.format_exc())
+        return error_sentinel
+
+    def unpickle(self, binary):
+        try:
+            return pickle.loads(binary)
+        except Exception as e:
+            self.log.error(f"Error deserializing binary: {e}")
+            self.log.trace(f"Offending binary: {binary}")
+            self.log.trace(traceback.format_exc())
+        return error_sentinel
+
+    def check_error(self, message):
+        if message is error_sentinel:
+            return True
+        if isinstance(message, dict) and len(message) == 1 and "_e" in message:
+            error, trace = message["_e"]
+            self.log.error(error)
+            self.log.trace(trace)
+            return True
+        return False
+
+
+class EngineClient(EngineBase):
 
     SERVER_CLASS = None
 
     def __init__(self, **kwargs):
+        super().__init__()
         self.name = f"EngineClient {self.__class__.__name__}"
         if self.SERVER_CLASS is None:
             raise ValueError(f"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}")
         self.CMDS = dict(self.SERVER_CLASS.CMDS)
         for k, v in list(self.CMDS.items()):
             self.CMDS[v] = k
-        self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}")
         self.socket_address = f"zmq_{rand_string(8)}.sock"
         self.socket_path = Path(tempfile.gettempdir()) / self.socket_address
         self.server_kwargs = kwargs.pop("server_kwargs", {})
@@ -38,10 +73,12 @@ def __init__(self, **kwargs):
     async def run_and_return(self, command, **kwargs):
         async with self.new_socket() as socket:
             message = self.make_message(command, args=kwargs)
+            if message is error_sentinel:
+                return
             await socket.send(message)
             binary = await socket.recv()
         # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}")
-        message = pickle.loads(binary)
+        message = self.unpickle(binary)
         self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}")
         # error handling
         if self.check_error(message):
@@ -50,26 +87,20 @@ async def run_and_return(self, command, **kwargs):
 
     async def run_and_yield(self, command, **kwargs):
         message = self.make_message(command, args=kwargs)
+        if message is error_sentinel:
+            return
         async with self.new_socket() as socket:
             await socket.send(message)
             while 1:
                 binary = await socket.recv()
                 # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}")
-                message = pickle.loads(binary)
+                message = self.unpickle(binary)
                 self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}")
                 # error handling
                 if self.check_error(message) or self.check_stop(message):
                     break
                 yield message
 
-    def check_error(self, message):
-        if isinstance(message, dict) and len(message) == 1 and "_e" in message:
-            error, trace = message["_e"]
-            self.log.error(error)
-            self.log.trace(trace)
-            return True
-        return False
-
     def check_stop(self, message):
         if isinstance(message, dict) and len(message) == 1 and "_s" in message:
             return True
@@ -130,12 +161,12 @@ def cleanup(self):
         self.socket_path.unlink(missing_ok=True)
 
 
-class EngineServer:
+class EngineServer(EngineBase):
 
     CMDS = {}
 
     def __init__(self, socket_path):
-        self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}")
+        super().__init__()
         self.name = f"EngineServer {self.__class__.__name__}"
         if socket_path is not None:
             # create ZeroMQ context
@@ -178,8 +209,10 @@ async def worker(self):
         try:
             while 1:
                 client_id, binary = await self.socket.recv_multipart()
-                message = pickle.loads(binary)
+                message = self.unpickle(binary)
                 self.log.debug(f"{self.name} got message: {message}")
+                if self.check_error(message):
+                    continue
 
                 cmd = message.get("c", None)
                 if not isinstance(cmd, int):

From ce3bd7bdf1fa2092807d0c1bbc5e1e3d6a49a9ea Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 6 May 2024 04:25:23 +0000
Subject: [PATCH 073/220] Bump werkzeug from 3.0.2 to 3.0.3

Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.2 to 3.0.3.
- [Release notes](https://github.com/pallets/werkzeug/releases)
- [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/werkzeug/compare/3.0.2...3.0.3)

---
updated-dependencies:
- dependency-name: werkzeug
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index d47650bf4..1f6d2500e 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -942,6 +942,7 @@ files = [
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"},
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"},
     {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"},
+    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"},
     {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"},
@@ -966,7 +967,6 @@ files = [
     {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8"},
     {file = "lxml-5.2.1-cp37-cp37m-win32.whl", hash = "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd"},
     {file = "lxml-5.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c"},
-    {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5"},
     {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b"},
     {file = "lxml-5.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a"},
     {file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585"},
@@ -2109,6 +2109,7 @@ optional = false
 python-versions = "*"
 files = [
     {file = "requests-file-2.0.0.tar.gz", hash = "sha256:20c5931629c558fda566cacc10cfe2cd502433e628f568c34c80d96a0cc95972"},
+    {file = "requests_file-2.0.0-py2.py3-none-any.whl", hash = "sha256:3e493d390adb44aa102ebea827a48717336d5268968c370eaf19abaf5cae13bf"},
 ]
 
 [package.dependencies]
@@ -2451,13 +2452,13 @@ files = [
 
 [[package]]
 name = "werkzeug"
-version = "3.0.2"
+version = "3.0.3"
 description = "The comprehensive WSGI web application library."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "werkzeug-3.0.2-py3-none-any.whl", hash = "sha256:3aac3f5da756f93030740bc235d3e09449efcf65f2f55e3602e1d851b8f48795"},
-    {file = "werkzeug-3.0.2.tar.gz", hash = "sha256:e39b645a6ac92822588e7b39a692e7828724ceae0b0d702ef96701f90e70128d"},
+    {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"},
+    {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"},
 ]
 
 [package.dependencies]

From f80ccac1d09d3cd23f57161ec41facb289f39ff9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 6 May 2024 12:59:05 -0400
Subject: [PATCH 074/220] misc tweaks

---
 bbot/core/event/base.py                     |   3 +-
 bbot/modules/base.py                        |   6 +-
 bbot/modules/internal/dns.py                |   2 +-
 bbot/modules/output/base.py                 |   1 +
 bbot/scanner/scanner.py                     |   2 +-
 bbot/test/test_step_1/test_dns.py           | 104 ++++++++++++++------
 bbot/test/test_step_1/test_modules_basic.py |   1 -
 7 files changed, 85 insertions(+), 34 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ff0af03ac..ac9ca654d 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -631,6 +631,7 @@ def json(self, mode="json", siem_friendly=False):
         j["timestamp"] = self.timestamp.timestamp()
         if self.host:
             j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
+            j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
         source_id = self.source_id
         if source_id:
             j["source"] = source_id
@@ -645,7 +646,7 @@ def json(self, mode="json", siem_friendly=False):
         for k, v in list(j.items()):
             if k == "data":
                 continue
-            if type(v) not in (str, int, float, bool, list, type(None)):
+            if type(v) not in (str, int, float, bool, list, dict, type(None)):
                 try:
                     j[k] = json.dumps(v, sort_keys=True)
                 except Exception:
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 6c048c9a2..bf8d5b8fe 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -112,6 +112,7 @@ class BaseModule:
     _name = "base"
     _type = "scan"
     _intercept = False
+    _shuffle_incoming_queue = True
 
     def __init__(self, scan):
         """Initializes a module instance.
@@ -1089,7 +1090,10 @@ def config(self):
     @property
     def incoming_event_queue(self):
         if self._incoming_event_queue is None:
-            self._incoming_event_queue = ShuffleQueue()
+            if self._shuffle_incoming_queue:
+                self._incoming_event_queue = ShuffleQueue()
+            else:
+                self._incoming_event_queue = asyncio.Queue()
         return self._incoming_event_queue
 
     @property
diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index 29e060626..f501fb08f 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -220,7 +220,7 @@ async def handle_wildcard_event(self, event):
                 event.add_tag(f"{rdtype.lower()}-{wildcard_tag}")
 
             # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
-            if wildcard_rdtypes:
+            if wildcard_rdtypes and not "target" in event.tags:
                 # these are the rdtypes that successfully resolve
                 resolved_rdtypes = set([c.upper() for c in event.dns_children])
                 # these are the rdtypes that have wildcards
diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index a934ec009..8a6eba9eb 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -8,6 +8,7 @@ class BaseOutputModule(BaseModule):
     _type = "output"
     scope_distance_modifier = None
     _stats_exclude = True
+    _shuffle_incoming_queue = False
 
     def human_event_str(self, event):
         event_type = f"[{event.type}]"
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 2dbd46569..3beb98252 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -547,7 +547,7 @@ def queued_event_types(self):
             queues.add(module.outgoing_event_queue)
 
         for q in queues:
-            for item in q._queue:
+            for item in getattr(q, "_queue", []):
                 try:
                     event, _ = item
                 except ValueError:
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index aa2a27907..0e8ca141b 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -222,44 +222,90 @@ async def test_wildcards(bbot_scanner):
     from bbot.scanner import Scanner
 
     # test with full scan
-    scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dns_resolution": True})
+    scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", whitelist=["github.io"], config={"dns_resolution": True})
+    await scan2._prep()
+    other_event = scan2.make_event(
+        "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dns"], source=scan2.root_event
+    )
+    await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
-    assert len(events) == 2
+    assert len(events) == 3
     assert 1 == len([e for e in events if e.type == "SCAN"])
+    unmodified_wildcard_events = [
+        e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io"
+    ]
+    assert len(unmodified_wildcard_events) == 1
+    assert unmodified_wildcard_events[0].tags == {
+        "a-record",
+        "target",
+        "aaaa-wildcard",
+        "in-scope",
+        "subdomain",
+        "cdn-github",
+        "aaaa-record",
+        "wildcard",
+        "a-wildcard",
+    }
+    modified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and e.data == "_wildcard.github.io"]
+    assert len(modified_wildcard_events) == 1
+    assert modified_wildcard_events[0].tags == {
+        "a-record",
+        "aaaa-wildcard",
+        "in-scope",
+        "subdomain",
+        "cdn-github",
+        "aaaa-record",
+        "wildcard",
+        "a-wildcard",
+    }
+    assert modified_wildcard_events[0].host_original == "lkjg.sdfgsg.jgkhajshdsadf.github.io"
+
+    # test with full scan (wildcard detection disabled for domain)
+    scan2 = Scanner(
+        "asdfl.gashdgkjsadgsdf.github.io",
+        whitelist=["github.io"],
+        config={"dns_wildcard_ignore": ["github.io"], "dns_resolution": True},
+    )
+    await scan2._prep()
+    other_event = scan2.make_event(
+        "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dns"], source=scan2.root_event
+    )
+    await scan2.ingress_module.queue_event(other_event, {})
+    events = [e async for e in scan2.async_start()]
+    assert len(events) == 3
+    assert 1 == len([e for e in events if e.type == "SCAN"])
+    unmodified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and "_wildcard" not in e.data]
+    assert len(unmodified_wildcard_events) == 2
     assert 1 == len(
         [
             e
-            for e in events
-            if e.type == "DNS_NAME"
-            and e.data == "_wildcard.github.io"
-            and all(
-                t in e.tags
-                for t in (
-                    "a-record",
-                    "target",
-                    "aaaa-wildcard",
-                    "in-scope",
-                    "subdomain",
-                    "aaaa-record",
-                    "wildcard",
-                    "a-wildcard",
-                )
-            )
+            for e in unmodified_wildcard_events
+            if e.data == "asdfl.gashdgkjsadgsdf.github.io"
+            and e.tags
+            == {
+                "target",
+                "a-record",
+                "in-scope",
+                "subdomain",
+                "cdn-github",
+                "aaaa-record",
+            }
         ]
     )
-
-    # test with full scan (wildcard detection disabled for domain)
-    scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", config={"dns_wildcard_ignore": ["github.io"]})
-    events = [e async for e in scan2.async_start()]
-    assert len(events) == 2
-    assert 1 == len([e for e in events if e.type == "SCAN"])
     assert 1 == len(
         [
             e
-            for e in events
-            if e.type == "DNS_NAME"
-            and e.data == "asdfl.gashdgkjsadgsdf.github.io"
-            and all(t in e.tags for t in ("a-record", "target", "in-scope", "subdomain", "aaaa-record"))
-            and not any(t in e.tags for t in ("wildcard", "a-wildcard", "aaaa-wildcard"))
+            for e in unmodified_wildcard_events
+            if e.data == "lkjg.sdfgsg.jgkhajshdsadf.github.io"
+            and e.tags
+            == {
+                "a-record",
+                "in-scope",
+                "subdomain",
+                "cdn-github",
+                "aaaa-record",
+            }
         ]
     )
+    modified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and e.data == "_wildcard.github.io"]
+    assert len(modified_wildcard_events) == 0
diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 249953365..7f7618ef9 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -75,7 +75,6 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     # omitted event types should be rejected
     url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", source=scan.root_event)
     result, reason = base_output_module_2._event_precheck(url_unverified)
-    log.critical(f"{url_unverified} / {result} / {reason}")
     assert result == False
     assert reason == "its type is omitted in the config"
 

From 73caf0a96a729fbd0243f85b33895a4f3522c503 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Mon, 6 May 2024 17:18:00 -0400
Subject: [PATCH 075/220] adding paramminer extract test

---
 .../test_module_paramminer_headers.py         | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
index 7cc8114e6..4e287ead3 100644
--- a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
+++ b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
@@ -69,3 +69,49 @@ def check(self, module_test, events):
             and "[Paramminer] Header: [tracestate] Reasons: [body] Reflection: [False]" in e.data["description"]
             for e in events
         )
+
+
+class TestParamminer_Headers_extract(Paramminer_Headers):
+    headers_body = """
+    <html>
+    <title>the title</title>
+    <body>
+    <a href="/page?foo=AAAAAAAAAAAAAA">Click Me</a>
+    </body>
+    </html>
+    """
+
+    headers_body_match = """
+    <html>
+    <title>the title</title>
+    <body>
+    <a href="/page?foo=AAAAAAAAAAAAAA">Click Me</a>
+    <p>Secret param "foo" found with value: AAAAAAAAAAAAAA</p>
+    </body>
+    </html>
+    """
+
+    async def setup_after_prep(self, module_test):
+        module_test.scan.modules["paramminer_headers"].rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA"
+        module_test.monkeypatch.setattr(
+            helper.HttpCompare, "gen_cache_buster", lambda *args, **kwargs: {"AAAAAA": "1"}
+        )
+        expect_args = dict(headers={"foo": "AAAAAAAAAAAAAA"})
+        respond_args = {"response_data": self.headers_body_match}
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+        respond_args = {"response_data": self.headers_body}
+        module_test.set_expect_requests(respond_args=respond_args)
+
+    def check(self, module_test, events):
+        for e in events:
+
+            print(e)
+            print(e.type)
+        assert any(
+            e.type == "FINDING"
+            and "[Paramminer] Header: [foo] Reasons: [body] Reflection: [True]" in e.data["description"]
+            for e in events
+        )
+
+

From 8178cdb0db6a4fd0dfda636fa8c671c90e572f8d Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Mon, 6 May 2024 19:45:37 -0400
Subject: [PATCH 076/220] adding param validation

---
 bbot/core/helpers/misc.py                     | 50 ++++++++++++++++---
 bbot/modules/paramminer_headers.py            |  7 +--
 bbot/test/test_step_1/test_helpers.py         | 49 ++++++++++++++++++
 .../test_module_paramminer_headers.py         |  3 +-
 4 files changed, 96 insertions(+), 13 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index b77aaa2b7..f635daa4e 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -972,15 +972,21 @@ def extract_params_xml(xml_data):
     return tags
 
 
-def extract_params_html(html_data):
+def extract_params_html(html_data, compare_mode="getparam"):
     """
-    Extracts parameters from an HTML object, yielding them one at a time.
+    Extracts parameters from an HTML object, yielding them one at a time. This function filters
+    these parameters based on a specified mode that determines the type of validation
+    or comparison against rules for headers, GET parameters, or cookies. If no mode is specified,
+    it defaults to 'getparam', which is the least restrictive.
 
     Args:
         html_data (str): HTML-formatted string.
+        compare_mode (str, optional): The mode to compare extracted parameter names against.
+            Defaults to 'getparam'. Valid modes are 'header', 'getparam', 'cookie'.
 
     Yields:
-        str: A string containing the parameter found in HTML object.
+        str: A string containing the parameter found in the HTML object that meets the
+        criteria of the specified mode.
 
     Examples:
         >>> html_data = '''
@@ -996,20 +1002,46 @@ def extract_params_html(html_data):
         ... </html>
         ... '''
         >>> list(extract_params_html(html_data))
-        ['user', 'param2', 'param3']
+        ['user', 'param1', 'param2', 'param3']
     """
+
+    def validate_param(param, compare_mode):
+        if len(param) > 100:
+            return False
+
+        # Define valid characters for each mode based on RFCs
+        valid_chars = {
+            "header": "".join(
+                chr(c) for c in range(33, 127) if chr(c) not in '"(),;:\\'
+            ),  # HTTP headers exclude CTLs, SP, DQUOTE, comma, semicolon, and backslash
+            "getparam": "".join(
+                chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="
+            ),  # URI reserved characters should be avoided or percent-encoded
+            "cookie": "".join(
+                chr(c) for c in range(33, 127) if chr(c) not in ' ",;=\\'
+            ),  # Cookies exclude spaces, quotes, comma, semicolon, equals, and backslash
+        }
+
+        if compare_mode not in valid_chars:
+            raise ValueError(f"Invalid compare_mode: {compare_mode}")
+
+        allowed_chars = set(valid_chars[compare_mode])
+        return set(param).issubset(allowed_chars)
+
     input_tag = bbot_regexes.input_tag_regex.findall(html_data)
 
     for i in input_tag:
         log.debug(f"FOUND PARAM ({i}) IN INPUT TAGS")
-        yield i
+        if validate_param(i, compare_mode):
+            yield i
 
     # check for jquery get parameters
     jquery_get = bbot_regexes.jquery_get_regex.findall(html_data)
 
     for i in jquery_get:
         log.debug(f"FOUND PARAM ({i}) IN JQUERY GET PARAMS")
-        yield i
+        if validate_param(i, compare_mode):
+            yield i
 
     # check for jquery post parameters
     jquery_post = bbot_regexes.jquery_post_regex.findall(html_data)
@@ -1018,12 +1050,14 @@ def extract_params_html(html_data):
             for x in i.split(","):
                 s = x.split(":")[0].rstrip()
                 log.debug(f"FOUND PARAM ({s}) IN A JQUERY POST PARAMS")
-                yield s
+                if validate_param(s, compare_mode):
+                    yield s
 
     a_tag = bbot_regexes.a_tag_regex.findall(html_data)
     for s in a_tag:
         log.debug(f"FOUND PARAM ({s}) IN A TAG GET PARAMS")
-        yield s
+        if validate_param(s, compare_mode):
+            yield s
 
 
 def extract_words(data, acronyms=True, wordninja=True, model=None, max_length=100, word_regexes=None):
diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index c7418912f..23955d8a8 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -199,11 +199,11 @@ def load_extracted_words(self, body, content_type):
         if not body:
             return None
         if content_type and "json" in content_type.lower():
-            return extract_params_json(body)
+            return extract_params_json(body, self.compare_mode)
         elif content_type and "xml" in content_type.lower():
-            return extract_params_xml(body)
+            return extract_params_xml(body, self.compare_mode)
         else:
-            return set(extract_params_html(body))
+            return set(extract_params_html(body, self.compare_mode))
 
     async def binary_search(self, compare_helper, url, group, reasons=None, reflection=False):
         if reasons is None:
@@ -249,4 +249,5 @@ async def finish(self):
                 results = await self.do_mining(untested_matches_copy, url, batch_size, compare_helper)
             except HttpCompareError as e:
                 self.debug(f"Encountered HttpCompareError: [{e}] for URL [{url}]")
+                continue
             await self.process_results(event, results)
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index c8045e595..70cd090c4 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -745,3 +745,52 @@ def test_liststring_invalidfnchars(helpers):
     with pytest.raises(ValueError) as e:
         helpers.parse_list_string("hello,world,bbot|test")
     assert str(e.value) == "Invalid character in string: bbot|test"
+
+
+# test extract_params_html
+def test_extract_params_html(helpers):
+    html_tests = """
+    <html>
+    <head>
+        <title>Test Links for Parameter Names</title>
+    </head>
+    <body>
+        <!-- Universal Valid: All parameter names should pass -->
+        <a href="/validPath?name=123&age=456">Universal Valid</a>
+
+        <!-- Mixed Validity: Different rules for headers, GET parameters, and cookies -->
+        <a href="/test?valid_name=1&valid-name=2&invalid,name=3">Mixed Validity</a>
+        <a href="/test?session_token=1&user.id=2&auth-token=3">Token Examples</a>
+
+        <!-- Valid for GET and Cookies, questionable for headers due to dashes or dots -->
+        <a href="/details?user-name=1&session-token=2">Common Web Names</a>
+        <a href="/info?client.id=1&access_token=2">API Style Names</a>
+
+        <!-- Bad examples that should fail all validations -->
+        <a href="/badPath?this_parameter_name_is_way_too_long_to_be_practical_but_hey_its_still_technically_valid=foo">Terrible Names</a>
+    </body>
+    </html>
+    """
+    extract_results = helpers.extract_params_html(html_tests)
+    valid_params = {"name", "valid_name", "session_token", "user-name", "client.id"}
+    invalid_params = {
+        "valid-name",
+        "invalid,name",
+        "auth-token",
+        "access_token",
+        "<script>",
+        "###$$$",
+        "this_parameter_name_is_way_too_long_to_be_practical_but_hey_look_its_still_technically_valid_yay",
+    }
+
+    extracted_params = set(
+        extract_results
+    )  # Assuming `extract_results` is a list of parameter names extracted from the HTML.
+
+    # Check that all valid parameters are present
+    for expected_param in valid_params:
+        assert expected_param in extracted_params, f"Missing expected parameter: {expected_param}"
+
+    # Check that no invalid parameters are present
+    for bad_param in invalid_params:
+        assert bad_param not in extracted_params, f"Invalid parameter found: {bad_param}"
diff --git a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
index 4e287ead3..ba569b513 100644
--- a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
+++ b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
@@ -86,6 +86,7 @@ class TestParamminer_Headers_extract(Paramminer_Headers):
     <title>the title</title>
     <body>
     <a href="/page?foo=AAAAAAAAAAAAAA">Click Me</a>
+    <a href="/page?foo=http://thisisjunk.com?whatever=value">Click Me</a>
     <p>Secret param "foo" found with value: AAAAAAAAAAAAAA</p>
     </body>
     </html>
@@ -113,5 +114,3 @@ def check(self, module_test, events):
             and "[Paramminer] Header: [foo] Reasons: [body] Reflection: [True]" in e.data["description"]
             for e in events
         )
-
-

From 6702e270946c27c12d810fa78d27c8babad14b98 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Mon, 6 May 2024 22:34:32 -0400
Subject: [PATCH 077/220] cleanup

---
 .../test_step_2/module_tests/test_module_paramminer_headers.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
index ba569b513..9cfb52312 100644
--- a/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
+++ b/bbot/test/test_step_2/module_tests/test_module_paramminer_headers.py
@@ -105,10 +105,7 @@ async def setup_after_prep(self, module_test):
         module_test.set_expect_requests(respond_args=respond_args)
 
     def check(self, module_test, events):
-        for e in events:
 
-            print(e)
-            print(e.type)
         assert any(
             e.type == "FINDING"
             and "[Paramminer] Header: [foo] Reasons: [body] Reflection: [True]" in e.data["description"]

From f0960bb42bf3632d58f6f4aaa3d0e064715b3be6 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 7 May 2024 00:09:52 -0400
Subject: [PATCH 078/220] better error handling in process pool .run()

---
 bbot/core/core.py                     |  29 +-------
 bbot/core/engine.py                   |   1 +
 bbot/core/helpers/misc.py             |  21 ++++++
 bbot/core/helpers/names_generator.py  |   3 +
 bbot/core/helpers/web.py              |   8 +-
 bbot/scanner/scanner.py               |   6 +-
 bbot/test/test_step_1/test_helpers.py |  31 ++++++++
 poetry.lock                           | 102 +++++++++++++++++++++++++-
 pyproject.toml                        |   1 +
 9 files changed, 170 insertions(+), 32 deletions(-)

diff --git a/bbot/core/core.py b/bbot/core/core.py
index d808f810f..23f611898 100644
--- a/bbot/core/core.py
+++ b/bbot/core/core.py
@@ -1,10 +1,9 @@
 import logging
-import traceback
 from copy import copy
-import multiprocessing
 from pathlib import Path
 from omegaconf import OmegaConf
 
+
 DEFAULT_CONFIG = None
 
 
@@ -22,27 +21,6 @@ class BBOTCore:
     - load quickly
     """
 
-    class BBOTProcess(multiprocessing.Process):
-
-        def __init__(self, *args, **kwargs):
-            self.logging_queue = kwargs.pop("logging_queue")
-            self.log_level = kwargs.pop("log_level")
-            super().__init__(*args, **kwargs)
-
-        def run(self):
-            log = logging.getLogger("bbot.core.process")
-            try:
-                from bbot.core import CORE
-
-                CORE.logger.setup_queue_handler(self.logging_queue, self.log_level)
-                super().run()
-            except KeyboardInterrupt:
-                log.warning(f"Got KeyboardInterrupt in {self.name}")
-                log.trace(traceback.format_exc())
-            except BaseException as e:
-                log.warning(f"Error in {self.name}: {e}")
-                log.trace(traceback.format_exc())
-
     def __init__(self):
         self._logger = None
         self._files_config = None
@@ -165,8 +143,9 @@ def files_config(self):
         return self._files_config
 
     def create_process(self, *args, **kwargs):
-        process = self.BBOTProcess(*args, logging_queue=self.logger.queue, log_level=self.logger.log_level, **kwargs)
-        process.daemon = True
+        from .helpers.process import BBOTProcess
+
+        process = BBOTProcess(*args, **kwargs)
         return process
 
     @property
diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 269e7c645..ada0e1301 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -125,6 +125,7 @@ def start_server(self):
                 self.socket_path,
             ),
             kwargs=self.server_kwargs,
+            custom_name="bbot dnshelper",
         )
         process.start()
         return process
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 206fc50f0..808190e9a 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -2327,6 +2327,27 @@ def get_exception_chain(e):
     return exception_chain
 
 
+def in_exception_chain(e, exc_types):
+    """
+    Given an Exception and a list of Exception types, returns whether any of the specified types are contained anywhere in the Exception chain.
+
+    Args:
+        e (BaseException): The exception to check
+        exc_types (list[Exception]): Exception types to consider intentional cancellations. Default is KeyboardInterrupt
+
+    Returns:
+        bool: Whether the error is the result of an intentional cancellaion
+
+    Examples:
+        >>> try:
+        ...     raise ValueError("This is a value error")
+        ... except Exception as e:
+        ...     if not in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
+        ...         raise
+    """
+    return any([isinstance(_, exc_types) for _ in get_exception_chain(e)])
+
+
 def get_traceback_details(e):
     """
     Retrieves detailed information from the traceback of an exception.
diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py
index 3e16b446a..41e1dd0ea 100644
--- a/bbot/core/helpers/names_generator.py
+++ b/bbot/core/helpers/names_generator.py
@@ -2,6 +2,7 @@
 
 adjectives = [
     "abnormal",
+    "accidental",
     "acoustic",
     "acrophobic",
     "adorable",
@@ -9,6 +10,7 @@
     "affectionate",
     "aggravated",
     "aggrieved",
+    "almighty",
     "anal",
     "atrocious",
     "awkward",
@@ -140,6 +142,7 @@
     "medicated",
     "mediocre",
     "melodramatic",
+    "mighty",
     "moist",
     "molten",
     "monstrous",
diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py
index 26773bc9c..cbc049fc1 100644
--- a/bbot/core/helpers/web.py
+++ b/bbot/core/helpers/web.py
@@ -693,8 +693,12 @@ async def _acatch(self, url, raise_error):
                 log.trace(traceback.format_exc())
         except BaseException as e:
             # don't log if the error is the result of an intentional cancellation
-            if not any(
-                isinstance(_e, asyncio.exceptions.CancelledError) for _e in self.parent_helper.get_exception_chain(e)
+            if not self.parent_helper.in_exception_chain(
+                e,
+                (
+                    KeyboardInterrupt,
+                    asyncio.CancelledError,
+                ),
             ):
                 log.trace(f"Unhandled exception with request to URL: {url}: {e}")
                 log.trace(traceback.format_exc())
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 3beb98252..ee17ef6a7 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -338,8 +338,7 @@ async def async_start(self):
             failed = False
 
         except BaseException as e:
-            exception_chain = self.helpers.get_exception_chain(e)
-            if any(isinstance(exc, (KeyboardInterrupt, asyncio.CancelledError)) for exc in exception_chain):
+            if self.helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
                 self.stop()
                 failed = False
             else:
@@ -1131,8 +1130,7 @@ def _handle_exception(self, e, context="scan", finally_callback=None, unhandled_
         if callable(context):
             context = f"{context.__qualname__}()"
         filename, lineno, funcname = self.helpers.get_traceback_details(e)
-        exception_chain = self.helpers.get_exception_chain(e)
-        if any(isinstance(exc, KeyboardInterrupt) for exc in exception_chain):
+        if self.helpers.in_exception_chain(e, (KeyboardInterrupt,)):
             log.debug(f"Interrupted")
             self.stop()
         elif isinstance(e, BrokenPipeError):
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index 4e3f3993e..ec2922c7a 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -451,6 +451,37 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
         < first_frequencies["e"]
     )
 
+    # error handling helpers
+    test_ran = False
+    try:
+        try:
+            raise KeyboardInterrupt("asdf")
+        except KeyboardInterrupt:
+            raise ValueError("asdf")
+    except Exception as e:
+        assert len(helpers.get_exception_chain(e)) == 2
+        assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, KeyboardInterrupt)]) == 1
+        assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, ValueError)]) == 1
+        assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) == True
+        assert helpers.in_exception_chain(e, (TypeError, OSError)) == False
+        test_ran = True
+    assert test_ran
+    test_ran = False
+    try:
+        try:
+            raise AttributeError("asdf")
+        except AttributeError:
+            raise ValueError("asdf")
+    except Exception as e:
+        assert len(helpers.get_exception_chain(e)) == 2
+        assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, AttributeError)]) == 1
+        assert len([_ for _ in helpers.get_exception_chain(e) if isinstance(_, ValueError)]) == 1
+        assert helpers.in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)) == False
+        assert helpers.in_exception_chain(e, (KeyboardInterrupt, AttributeError)) == True
+        assert helpers.in_exception_chain(e, (AttributeError,)) == True
+        test_ran = True
+    assert test_ran
+
 
 def test_word_cloud(helpers, bbot_scanner):
     number_mutations = helpers.word_cloud.get_number_mutations("base2_p013", n=5, padding=2)
diff --git a/poetry.lock b/poetry.lock
index be6fea410..6d578e6b5 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2249,6 +2249,106 @@ lint = ["black", "flake8", "isort", "mypy", "types-requests"]
 release = ["build", "towncrier", "twine"]
 test = ["commentjson", "packaging", "pytest"]
 
+[[package]]
+name = "setproctitle"
+version = "1.3.3"
+description = "A Python module to customize the process title"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "setproctitle-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:897a73208da48db41e687225f355ce993167079eda1260ba5e13c4e53be7f754"},
+    {file = "setproctitle-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8c331e91a14ba4076f88c29c777ad6b58639530ed5b24b5564b5ed2fd7a95452"},
+    {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbbd6c7de0771c84b4aa30e70b409565eb1fc13627a723ca6be774ed6b9d9fa3"},
+    {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c05ac48ef16ee013b8a326c63e4610e2430dbec037ec5c5b58fcced550382b74"},
+    {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1342f4fdb37f89d3e3c1c0a59d6ddbedbde838fff5c51178a7982993d238fe4f"},
+    {file = "setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc74e84fdfa96821580fb5e9c0b0777c1c4779434ce16d3d62a9c4d8c710df39"},
+    {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9617b676b95adb412bb69645d5b077d664b6882bb0d37bfdafbbb1b999568d85"},
+    {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6a249415f5bb88b5e9e8c4db47f609e0bf0e20a75e8d744ea787f3092ba1f2d0"},
+    {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:38da436a0aaace9add67b999eb6abe4b84397edf4a78ec28f264e5b4c9d53cd5"},
+    {file = "setproctitle-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:da0d57edd4c95bf221b2ebbaa061e65b1788f1544977288bdf95831b6e44e44d"},
+    {file = "setproctitle-1.3.3-cp310-cp310-win32.whl", hash = "sha256:a1fcac43918b836ace25f69b1dca8c9395253ad8152b625064415b1d2f9be4fb"},
+    {file = "setproctitle-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:200620c3b15388d7f3f97e0ae26599c0c378fdf07ae9ac5a13616e933cbd2086"},
+    {file = "setproctitle-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:334f7ed39895d692f753a443102dd5fed180c571eb6a48b2a5b7f5b3564908c8"},
+    {file = "setproctitle-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:950f6476d56ff7817a8fed4ab207727fc5260af83481b2a4b125f32844df513a"},
+    {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:195c961f54a09eb2acabbfc90c413955cf16c6e2f8caa2adbf2237d1019c7dd8"},
+    {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f05e66746bf9fe6a3397ec246fe481096664a9c97eb3fea6004735a4daf867fd"},
+    {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5901a31012a40ec913265b64e48c2a4059278d9f4e6be628441482dd13fb8b5"},
+    {file = "setproctitle-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64286f8a995f2cd934082b398fc63fca7d5ffe31f0e27e75b3ca6b4efda4e353"},
+    {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:184239903bbc6b813b1a8fc86394dc6ca7d20e2ebe6f69f716bec301e4b0199d"},
+    {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:664698ae0013f986118064b6676d7dcd28fefd0d7d5a5ae9497cbc10cba48fa5"},
+    {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e5119a211c2e98ff18b9908ba62a3bd0e3fabb02a29277a7232a6fb4b2560aa0"},
+    {file = "setproctitle-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:417de6b2e214e837827067048f61841f5d7fc27926f2e43954567094051aff18"},
+    {file = "setproctitle-1.3.3-cp311-cp311-win32.whl", hash = "sha256:6a143b31d758296dc2f440175f6c8e0b5301ced3b0f477b84ca43cdcf7f2f476"},
+    {file = "setproctitle-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a680d62c399fa4b44899094027ec9a1bdaf6f31c650e44183b50d4c4d0ccc085"},
+    {file = "setproctitle-1.3.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d4460795a8a7a391e3567b902ec5bdf6c60a47d791c3b1d27080fc203d11c9dc"},
+    {file = "setproctitle-1.3.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bdfd7254745bb737ca1384dee57e6523651892f0ea2a7344490e9caefcc35e64"},
+    {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477d3da48e216d7fc04bddab67b0dcde633e19f484a146fd2a34bb0e9dbb4a1e"},
+    {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ab2900d111e93aff5df9fddc64cf51ca4ef2c9f98702ce26524f1acc5a786ae7"},
+    {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:088b9efc62d5aa5d6edf6cba1cf0c81f4488b5ce1c0342a8b67ae39d64001120"},
+    {file = "setproctitle-1.3.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6d50252377db62d6a0bb82cc898089916457f2db2041e1d03ce7fadd4a07381"},
+    {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:87e668f9561fd3a457ba189edfc9e37709261287b52293c115ae3487a24b92f6"},
+    {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:287490eb90e7a0ddd22e74c89a92cc922389daa95babc833c08cf80c84c4df0a"},
+    {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:4fe1c49486109f72d502f8be569972e27f385fe632bd8895f4730df3c87d5ac8"},
+    {file = "setproctitle-1.3.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4a6ba2494a6449b1f477bd3e67935c2b7b0274f2f6dcd0f7c6aceae10c6c6ba3"},
+    {file = "setproctitle-1.3.3-cp312-cp312-win32.whl", hash = "sha256:2df2b67e4b1d7498632e18c56722851ba4db5d6a0c91aaf0fd395111e51cdcf4"},
+    {file = "setproctitle-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:f38d48abc121263f3b62943f84cbaede05749047e428409c2c199664feb6abc7"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:816330675e3504ae4d9a2185c46b573105d2310c20b19ea2b4596a9460a4f674"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68f960bc22d8d8e4ac886d1e2e21ccbd283adcf3c43136161c1ba0fa509088e0"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e6e7adff74796ef12753ff399491b8827f84f6c77659d71bd0b35870a17d8f"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53bc0d2358507596c22b02db079618451f3bd720755d88e3cccd840bafb4c41c"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad6d20f9541f5f6ac63df553b6d7a04f313947f550eab6a61aa758b45f0d5657"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c1c84beab776b0becaa368254801e57692ed749d935469ac10e2b9b825dbdd8e"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:507e8dc2891021350eaea40a44ddd887c9f006e6b599af8d64a505c0f718f170"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:b1067647ac7aba0b44b591936118a22847bda3c507b0a42d74272256a7a798e9"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2e71f6365744bf53714e8bd2522b3c9c1d83f52ffa6324bd7cbb4da707312cd8"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:7f1d36a1e15a46e8ede4e953abb104fdbc0845a266ec0e99cc0492a4364f8c44"},
+    {file = "setproctitle-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c9a402881ec269d0cc9c354b149fc29f9ec1a1939a777f1c858cdb09c7a261df"},
+    {file = "setproctitle-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ff814dea1e5c492a4980e3e7d094286077054e7ea116cbeda138819db194b2cd"},
+    {file = "setproctitle-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:accb66d7b3ccb00d5cd11d8c6e07055a4568a24c95cf86109894dcc0c134cc89"},
+    {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554eae5a5b28f02705b83a230e9d163d645c9a08914c0ad921df363a07cf39b1"},
+    {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a911b26264dbe9e8066c7531c0591cfab27b464459c74385b276fe487ca91c12"},
+    {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2982efe7640c4835f7355fdb4da313ad37fb3b40f5c69069912f8048f77b28c8"},
+    {file = "setproctitle-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df3f4274b80709d8bcab2f9a862973d453b308b97a0b423a501bcd93582852e3"},
+    {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:af2c67ae4c795d1674a8d3ac1988676fa306bcfa1e23fddb5e0bd5f5635309ca"},
+    {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:af4061f67fd7ec01624c5e3c21f6b7af2ef0e6bab7fbb43f209e6506c9ce0092"},
+    {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:37a62cbe16d4c6294e84670b59cf7adcc73faafe6af07f8cb9adaf1f0e775b19"},
+    {file = "setproctitle-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a83ca086fbb017f0d87f240a8f9bbcf0809f3b754ee01cec928fff926542c450"},
+    {file = "setproctitle-1.3.3-cp38-cp38-win32.whl", hash = "sha256:059f4ce86f8cc92e5860abfc43a1dceb21137b26a02373618d88f6b4b86ba9b2"},
+    {file = "setproctitle-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ab92e51cd4a218208efee4c6d37db7368fdf182f6e7ff148fb295ecddf264287"},
+    {file = "setproctitle-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c7951820b77abe03d88b114b998867c0f99da03859e5ab2623d94690848d3e45"},
+    {file = "setproctitle-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5bc94cf128676e8fac6503b37763adb378e2b6be1249d207630f83fc325d9b11"},
+    {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f5d9027eeda64d353cf21a3ceb74bb1760bd534526c9214e19f052424b37e42"},
+    {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e4a8104db15d3462e29d9946f26bed817a5b1d7a47eabca2d9dc2b995991503"},
+    {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c32c41ace41f344d317399efff4cffb133e709cec2ef09c99e7a13e9f3b9483c"},
+    {file = "setproctitle-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbf16381c7bf7f963b58fb4daaa65684e10966ee14d26f5cc90f07049bfd8c1e"},
+    {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e18b7bd0898398cc97ce2dfc83bb192a13a087ef6b2d5a8a36460311cb09e775"},
+    {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:69d565d20efe527bd8a9b92e7f299ae5e73b6c0470f3719bd66f3cd821e0d5bd"},
+    {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ddedd300cd690a3b06e7eac90ed4452348b1348635777ce23d460d913b5b63c3"},
+    {file = "setproctitle-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:415bfcfd01d1fbf5cbd75004599ef167a533395955305f42220a585f64036081"},
+    {file = "setproctitle-1.3.3-cp39-cp39-win32.whl", hash = "sha256:21112fcd2195d48f25760f0eafa7a76510871bbb3b750219310cf88b04456ae3"},
+    {file = "setproctitle-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:5a740f05d0968a5a17da3d676ce6afefebeeeb5ce137510901bf6306ba8ee002"},
+    {file = "setproctitle-1.3.3-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6b9e62ddb3db4b5205c0321dd69a406d8af9ee1693529d144e86bd43bcb4b6c0"},
+    {file = "setproctitle-1.3.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e3b99b338598de0bd6b2643bf8c343cf5ff70db3627af3ca427a5e1a1a90dd9"},
+    {file = "setproctitle-1.3.3-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ae9a02766dad331deb06855fb7a6ca15daea333b3967e214de12cfae8f0ef5"},
+    {file = "setproctitle-1.3.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:200ede6fd11233085ba9b764eb055a2a191fb4ffb950c68675ac53c874c22e20"},
+    {file = "setproctitle-1.3.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0d3a953c50776751e80fe755a380a64cb14d61e8762bd43041ab3f8cc436092f"},
+    {file = "setproctitle-1.3.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5e08e232b78ba3ac6bc0d23ce9e2bee8fad2be391b7e2da834fc9a45129eb87"},
+    {file = "setproctitle-1.3.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1da82c3e11284da4fcbf54957dafbf0655d2389cd3d54e4eaba636faf6d117a"},
+    {file = "setproctitle-1.3.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:aeaa71fb9568ebe9b911ddb490c644fbd2006e8c940f21cb9a1e9425bd709574"},
+    {file = "setproctitle-1.3.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:59335d000c6250c35989394661eb6287187854e94ac79ea22315469ee4f4c244"},
+    {file = "setproctitle-1.3.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3ba57029c9c50ecaf0c92bb127224cc2ea9fda057b5d99d3f348c9ec2855ad3"},
+    {file = "setproctitle-1.3.3-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d876d355c53d975c2ef9c4f2487c8f83dad6aeaaee1b6571453cb0ee992f55f6"},
+    {file = "setproctitle-1.3.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:224602f0939e6fb9d5dd881be1229d485f3257b540f8a900d4271a2c2aa4e5f4"},
+    {file = "setproctitle-1.3.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d7f27e0268af2d7503386e0e6be87fb9b6657afd96f5726b733837121146750d"},
+    {file = "setproctitle-1.3.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5e7266498cd31a4572378c61920af9f6b4676a73c299fce8ba93afd694f8ae7"},
+    {file = "setproctitle-1.3.3-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33c5609ad51cd99d388e55651b19148ea99727516132fb44680e1f28dd0d1de9"},
+    {file = "setproctitle-1.3.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:eae8988e78192fd1a3245a6f4f382390b61bce6cfcc93f3809726e4c885fa68d"},
+    {file = "setproctitle-1.3.3.tar.gz", hash = "sha256:c913e151e7ea01567837ff037a23ca8740192880198b7fbb90b16d181607caae"},
+]
+
+[package.extras]
+test = ["pytest"]
+
 [[package]]
 name = "setuptools"
 version = "69.5.1"
@@ -2637,4 +2737,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "ed8bb07e4ff5a5f665402db33f9016409547bef1ccb6a8c2c626c44fde075abb"
+content-hash = "5446b7a61ccdabf76f3d390cdfcb2904d3be142a0370bf2317dc8d2e3b99542a"
diff --git a/pyproject.toml b/pyproject.toml
index 627f323d8..95bd25704 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,6 +52,7 @@ regex = "^2024.4.16"
 unidecode = "^1.3.8"
 radixtarget = "^1.0.0.15"
 cloudcheck = "^5.0.0.350"
+setproctitle = "^1.3.3"
 
 [tool.poetry.group.dev.dependencies]
 flake8 = ">=6,<8"

From f9ae1e6ae86cf2be34209be1c49048e6bfb552a6 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 7 May 2024 00:47:10 -0400
Subject: [PATCH 079/220] tweaking regexes and improving tests

---
 bbot/core/helpers/misc.py             |  30 ++++---
 bbot/core/helpers/regexes.py          |   4 +-
 bbot/test/test_step_1/test_helpers.py | 109 ++++++++++++++++++++++----
 3 files changed, 114 insertions(+), 29 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index f635daa4e..bcef1f351 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -1012,7 +1012,7 @@ def validate_param(param, compare_mode):
         # Define valid characters for each mode based on RFCs
         valid_chars = {
             "header": "".join(
-                chr(c) for c in range(33, 127) if chr(c) not in '"(),;:\\'
+                chr(c) for c in range(33, 127) if chr(c) not in '."(),;:\\'
             ),  # HTTP headers exclude CTLs, SP, DQUOTE, comma, semicolon, and backslash
             "getparam": "".join(
                 chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="
@@ -1031,17 +1031,19 @@ def validate_param(param, compare_mode):
     input_tag = bbot_regexes.input_tag_regex.findall(html_data)
 
     for i in input_tag:
-        log.debug(f"FOUND PARAM ({i}) IN INPUT TAGS")
         if validate_param(i, compare_mode):
+            log.debug(f"FOUND PARAM ({i}) IN INPUT TAGS")
             yield i
 
     # check for jquery get parameters
     jquery_get = bbot_regexes.jquery_get_regex.findall(html_data)
-
-    for i in jquery_get:
-        log.debug(f"FOUND PARAM ({i}) IN JQUERY GET PARAMS")
-        if validate_param(i, compare_mode):
-            yield i
+    if jquery_get:
+        for i in jquery_get:
+            for x in i.split(","):
+                s = x.split(":")[0].rstrip()
+                if validate_param(s, compare_mode):
+                    log.debug(f"FOUND PARAM ({s}) IN A JQUERY GET PARAMS")
+                    yield s
 
     # check for jquery post parameters
     jquery_post = bbot_regexes.jquery_post_regex.findall(html_data)
@@ -1049,15 +1051,19 @@ def validate_param(param, compare_mode):
         for i in jquery_post:
             for x in i.split(","):
                 s = x.split(":")[0].rstrip()
-                log.debug(f"FOUND PARAM ({s}) IN A JQUERY POST PARAMS")
                 if validate_param(s, compare_mode):
+                    log.debug(f"FOUND PARAM ({s}) IN A JQUERY POST PARAMS")
                     yield s
 
     a_tag = bbot_regexes.a_tag_regex.findall(html_data)
-    for s in a_tag:
-        log.debug(f"FOUND PARAM ({s}) IN A TAG GET PARAMS")
-        if validate_param(s, compare_mode):
-            yield s
+    for tag in a_tag:
+        a_tag_querystring = tag.split("&") if tag else []
+        for s in a_tag_querystring:
+            if "=" in s:
+                s0 = s.split("=")[0]
+                if validate_param(s0, compare_mode):
+                    log.debug(f"FOUND PARAM ({s0}) IN A TAG GET PARAMS")
+                    yield s0
 
 
 def extract_words(data, acronyms=True, wordninja=True, model=None, max_length=100, word_regexes=None):
diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py
index 6e80801a6..31cfd3fdf 100644
--- a/bbot/core/helpers/regexes.py
+++ b/bbot/core/helpers/regexes.py
@@ -90,9 +90,9 @@
 
 # For use with extract_params_html helper
 input_tag_regex = re.compile(r"<input[^>]+?name=[\"\'](\w+)[\"\']")
-jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=")
+jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}")
 jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}")
-a_tag_regex = re.compile(r"<a[^>]*href=[\"\'][^\"\'?>]*\?([^&\"\'=]+)")
+a_tag_regex = re.compile(r"<a[^>]*href=[\"\'][^\"\'#>]*\?([^\"\'#>]+)[\"\']")
 
 valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+"
 
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index 70cd090c4..e8e09a892 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -752,7 +752,11 @@ def test_extract_params_html(helpers):
     html_tests = """
     <html>
     <head>
-        <title>Test Links for Parameter Names</title>
+        <title>Get extract</title>
+        <script>
+            $.get("/test", {jqueryget: "value1"});
+            $.post("/test", {jquerypost: "value2"});
+        </script>
     </head>
     <body>
         <!-- Universal Valid: All parameter names should pass -->
@@ -762,35 +766,110 @@ def test_extract_params_html(helpers):
         <a href="/test?valid_name=1&valid-name=2&invalid,name=3">Mixed Validity</a>
         <a href="/test?session_token=1&user.id=2&auth-token=3">Token Examples</a>
 
-        <!-- Valid for GET and Cookies, questionable for headers due to dashes or dots -->
-        <a href="/details?user-name=1&session-token=2">Common Web Names</a>
+        <!-- Valid for GET and Cookies, not valid for headers -->
+        <a href="/details?user-name=1&parens()=2">Common Web Names</a>
         <a href="/info?client.id=1&access_token=2">API Style Names</a>
 
         <!-- Bad examples that should fail all validations -->
-        <a href="/badPath?this_parameter_name_is_way_too_long_to_be_practical_but_hey_its_still_technically_valid=foo">Terrible Names</a>
+        <a href="/badPath?this_parameter_name_is_seriously_way_too_long_to_be_practical_but_hey_look_its_still_technically_valid_wow=foo">Invalid</a>
+        <a href="/badPath?###$$$=test">Invalid</a>
+        <a href="/badPath?<script>=test">Invalid</a>
+        <input name="abcd" value="zxyz>MixedaValidity</input>
     </body>
     </html>
     """
-    extract_results = helpers.extract_params_html(html_tests)
-    valid_params = {"name", "valid_name", "session_token", "user-name", "client.id"}
-    invalid_params = {
+    getparam_extract_results = set(helpers.extract_params_html(html_tests, "getparam"))
+    getparam_valid_params = {
+        "name",
+        "age",
+        "valid_name",
         "valid-name",
+        "session_token",
+        "user.id",
+        "user-name",
+        "client.id",
+        "auth-token",
+        "access_token",
+        "abcd",
+        "jqueryget",
+    }
+    getparam_invalid_params = {
         "invalid,name",
+        "<script>",
+        "###$$$",
+        "this_parameter_name_is_seriously_way_too_long_to_be_practical_but_hey_look_its_still_technically_valid_wow",
+        "parens()",
+    }
+    getparam_extracted_params = set(getparam_extract_results)
+
+    # Check that all valid parameters are present
+    for expected_param in getparam_valid_params:
+        assert expected_param in getparam_extracted_params, f"Missing expected parameter: {expected_param}"
+
+    # Check that no invalid parameters are present
+    for bad_param in getparam_invalid_params:
+        assert bad_param not in getparam_extracted_params, f"Invalid parameter found: {bad_param}"
+
+    header_extract_results = set(helpers.extract_params_html(html_tests, "header"))
+    header_valid_params = {
+        "name",
+        "age",
+        "valid_name",
+        "valid-name",
+        "session_token",
+        "user-name",
         "auth-token",
         "access_token",
+        "abcd",
+        "jqueryget",
+    }
+    header_invalid_params = {
+        "user.id",
+        "client.id",
+        "invalid,name",
         "<script>",
         "###$$$",
-        "this_parameter_name_is_way_too_long_to_be_practical_but_hey_look_its_still_technically_valid_yay",
+        "this_parameter_name_is_seriously_way_too_long_to_be_practical_but_hey_look_its_still_technically_valid_wow",
+        "parens()",
     }
+    header_extracted_params = set(header_extract_results)
+
+    # Check that all valid parameters are present
+    for expected_param in header_valid_params:
+        assert expected_param in header_extracted_params, f"Missing expected parameter: {expected_param}"
 
-    extracted_params = set(
-        extract_results
-    )  # Assuming `extract_results` is a list of parameter names extracted from the HTML.
+    # Check that no invalid parameters are present
+    for bad_param in header_invalid_params:
+        assert bad_param not in header_extracted_params, f"Invalid parameter found: {bad_param}"
+
+    cookie_extract_results = set(helpers.extract_params_html(html_tests, "cookie"))
+    cookie_valid_params = {
+        "name",
+        "age",
+        "valid_name",
+        "valid-name",
+        "session_token",
+        "user-name",
+        "auth-token",
+        "access_token",
+        "parens()",
+        "user.id",
+        "client.id",
+        "abcd",
+        "jqueryget",
+    }
+    cookie_invalid_params = {
+        "invalid,name",
+        "<script>",
+        "###$$$",
+        "this_parameter_name_is_seriously_way_too_long_to_be_practical_but_hey_look_its_still_technically_valid_wow",
+    }
+    cookie_extracted_params = set(cookie_extract_results)
 
     # Check that all valid parameters are present
-    for expected_param in valid_params:
-        assert expected_param in extracted_params, f"Missing expected parameter: {expected_param}"
+    for expected_param in cookie_valid_params:
+        assert expected_param in cookie_extracted_params, f"Missing expected parameter: {expected_param}"
 
     # Check that no invalid parameters are present
-    for bad_param in invalid_params:
-        assert bad_param not in extracted_params, f"Invalid parameter found: {bad_param}"
+    for bad_param in cookie_invalid_params:
+        assert bad_param not in cookie_extracted_params, f"Invalid parameter found: {bad_param}"

From ceb999ce449f3c075f00e5779ee3fecfa37c4ab2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 7 May 2024 09:04:06 -0400
Subject: [PATCH 080/220] add process.py

---
 bbot/core/helpers/process.py | 54 ++++++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 bbot/core/helpers/process.py

diff --git a/bbot/core/helpers/process.py b/bbot/core/helpers/process.py
new file mode 100644
index 000000000..90843b441
--- /dev/null
+++ b/bbot/core/helpers/process.py
@@ -0,0 +1,54 @@
+import logging
+import traceback
+import multiprocessing
+from multiprocessing.context import SpawnProcess
+
+from .misc import in_exception_chain
+
+
+current_process = multiprocessing.current_process()
+
+
+class BBOTProcess(SpawnProcess):
+
+    default_name = "bbot process pool"
+
+    def __init__(self, *args, **kwargs):
+        self.log_queue = kwargs.pop("log_queue", None)
+        self.log_level = kwargs.pop("log_level", None)
+        self.custom_name = kwargs.pop("custom_name", self.default_name)
+        super().__init__(*args, **kwargs)
+        self.daemon = True
+
+    def run(self):
+        """
+        A version of Process.run() with BBOT logging and better error handling
+        """
+        log = logging.getLogger("bbot.core.process")
+        try:
+            if self.log_level is not None and self.log_queue is not None:
+                from bbot.core import CORE
+
+                CORE.logger.setup_queue_handler(self.log_queue, self.log_level)
+            if self.custom_name:
+                from setproctitle import setproctitle
+
+                setproctitle(str(self.custom_name))
+            super().run()
+        except BaseException as e:
+            if not in_exception_chain(e, (KeyboardInterrupt,)):
+                log.warning(f"Error in {self.name}: {e}")
+            log.trace(traceback.format_exc())
+
+
+if current_process.name == "MainProcess":
+    # if this is the main bbot process, set the logger and queue for the first time
+    from bbot.core import CORE
+    from functools import partialmethod
+
+    BBOTProcess.__init__ = partialmethod(
+        BBOTProcess.__init__, log_level=CORE.logger.log_level, log_queue=CORE.logger.queue
+    )
+
+mp_context = multiprocessing.get_context("spawn")
+mp_context.Process = BBOTProcess

From 43058b10c3ad9ce54951dead9db059074f177308 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 7 May 2024 11:43:29 -0400
Subject: [PATCH 081/220] fixing validation for xml/json params

---
 bbot/core/helpers/misc.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index e87067d95..9cac1ca42 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -821,7 +821,7 @@ def truncate_string(s, n):
         return s
 
 
-def extract_params_json(json_data):
+def extract_params_json(json_data, compare_mode="getparam"):
     """
     Extracts keys from a JSON object and returns them as a set. Used by the `paramminer_headers` module.
 
@@ -851,18 +851,18 @@ def extract_params_json(json_data):
         current_data = stack.pop()
         if isinstance(current_data, dict):
             for key, value in current_data.items():
-                keys.add(key)
+                if _validate_param(key, compare_mode):
+                    keys.add(key)
                 if isinstance(value, (dict, list)):
                     stack.append(value)
         elif isinstance(current_data, list):
             for item in current_data:
                 if isinstance(item, (dict, list)):
                     stack.append(item)
-
     return keys
 
 
-def extract_params_xml(xml_data):
+def extract_params_xml(xml_data, compare_mode="getparam"):
     """
     Extracts tags from an XML object and returns them as a set.
 
@@ -892,7 +892,8 @@ def extract_params_xml(xml_data):
 
     while stack:
         current_element = stack.pop()
-        tags.add(current_element.tag)
+        if _validate_param(current_element.tag, compare_mode):
+            tags.add(current_element.tag)
         for child in current_element:
             stack.append(child)
     return tags

From e22c701060e88bd461e5198b5cf02449b944b5e3 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 7 May 2024 15:42:58 -0400
Subject: [PATCH 082/220] correcting error handling in finish()

---
 bbot/modules/paramminer_headers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index d1cef4b22..28c94b9b6 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -239,7 +239,7 @@ async def finish(self):
                 compare_helper = self.helpers.http_compare(url)
             except HttpCompareError as e:
                 self.debug(f"Error initializing compare helper: {e}")
-                return
+                continue
             untested_matches_copy = untested_matches.copy()
             for i in untested_matches:
                 h = hash(i + url)

From c8e9fcc4635e14604de860f6f3ef93e7dc7bd92a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 7 May 2024 17:22:44 -0400
Subject: [PATCH 083/220] more WIP http engine

---
 bbot/core/config/logger.py        |  16 ++---
 bbot/core/core.py                 |  12 +++-
 bbot/core/engine.py               |   7 +-
 bbot/core/helpers/web/engine.py   |  89 ++++---------------------
 bbot/core/helpers/web/web.py      |  74 ++++++++++-----------
 bbot/modules/httpx.py             |   3 +-
 bbot/scanner/preset/preset.py     |  86 ++++++------------------
 bbot/scanner/target.py            | 106 +++++++++++++++++++++++++++++-
 bbot/test/conftest.py             |   6 --
 bbot/test/test_step_1/test_web.py |   6 +-
 10 files changed, 202 insertions(+), 203 deletions(-)

diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py
index a187af141..6a213d42d 100644
--- a/bbot/core/config/logger.py
+++ b/bbot/core/config/logger.py
@@ -172,6 +172,13 @@ def include_logger(self, logger):
         for handler in self.log_handlers.values():
             self.add_log_handler(handler)
 
+    def stderr_filter(self, record):
+        if record.levelno == logging.TRACE and self.log_level > logging.DEBUG:
+            return False
+        if record.levelno < self.log_level:
+            return False
+        return True
+
     @property
     def log_handlers(self):
         if self._log_handlers is None:
@@ -189,16 +196,9 @@ def log_handlers(self):
                 f"{log_dir}/bbot.debug.log", when="d", interval=1, backupCount=14
             )
 
-            def stderr_filter(record):
-                if record.levelno == logging.TRACE and self.log_level > logging.DEBUG:
-                    return False
-                if record.levelno < self.log_level:
-                    return False
-                return True
-
             # Log to stderr
             stderr_handler = logging.StreamHandler(sys.stderr)
-            stderr_handler.addFilter(stderr_filter)
+            stderr_handler.addFilter(self.stderr_filter)
             # log to files
             debug_handler.addFilter(lambda x: x.levelno == logging.TRACE or (x.levelno < logging.VERBOSE))
             main_handler.addFilter(lambda x: x.levelno != logging.TRACE and x.levelno >= logging.VERBOSE)
diff --git a/bbot/core/core.py b/bbot/core/core.py
index 23f611898..0a789c0ac 100644
--- a/bbot/core/core.py
+++ b/bbot/core/core.py
@@ -1,3 +1,4 @@
+import os
 import logging
 from copy import copy
 from pathlib import Path
@@ -143,9 +144,16 @@ def files_config(self):
         return self._files_config
 
     def create_process(self, *args, **kwargs):
-        from .helpers.process import BBOTProcess
+        if os.environ.get("BBOT_TESTING", "") == "True":
+            import threading
 
-        process = BBOTProcess(*args, **kwargs)
+            kwargs.pop("custom_name", None)
+            process = threading.Thread(*args, **kwargs)
+        else:
+            from .helpers.process import BBOTProcess
+
+            process = BBOTProcess(*args, **kwargs)
+        process.daemon = True
         return process
 
     @property
diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index aab2898f8..9242630c2 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -60,6 +60,7 @@ class EngineClient(EngineBase):
     def __init__(self, **kwargs):
         super().__init__()
         self.name = f"EngineClient {self.__class__.__name__}"
+        self.process = None
         self.process_name = multiprocessing.current_process().name
         if self.SERVER_CLASS is None:
             raise ValueError(f"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}")
@@ -127,7 +128,7 @@ def available_commands(self):
 
     def start_server(self):
         if self.process_name == "MainProcess":
-            process = CORE.create_process(
+            self.process = CORE.create_process(
                 target=self.server_process,
                 args=(
                     self.SERVER_CLASS,
@@ -136,8 +137,8 @@ def start_server(self):
                 kwargs=self.server_kwargs,
                 custom_name="bbot dnshelper",
             )
-            process.start()
-            return process
+            self.process.start()
+            return self.process
         else:
             raise BBOTEngineError(f"Tried to start server from process {self.process_name}")
 
diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 3a13bbb2d..7b9260f29 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -4,87 +4,16 @@
 import asyncio
 import logging
 import traceback
-from httpx._models import Cookies
 from socksio.exceptions import SOCKSError
 from contextlib import asynccontextmanager
 
+from .client import BBOTAsyncClient
 from bbot.core.engine import EngineServer
 from bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain
 
 log = logging.getLogger("bbot.core.helpers.web.engine")
 
 
-class DummyCookies(Cookies):
-    def extract_cookies(self, *args, **kwargs):
-        pass
-
-
-class BBOTAsyncClient(httpx.AsyncClient):
-    """
-    A subclass of httpx.AsyncClient tailored with BBOT-specific configurations and functionalities.
-    This class provides rate limiting, logging, configurable timeouts, user-agent customization, custom
-    headers, and proxy settings. Additionally, it allows the disabling of cookies, making it suitable
-    for use across an entire scan.
-
-    Attributes:
-        _bbot_scan (object): BBOT scan object containing configuration details.
-        _persist_cookies (bool): Flag to determine whether cookies should be persisted across requests.
-
-    Examples:
-        >>> async with BBOTAsyncClient(_bbot_scan=bbot_scan_object) as client:
-        >>>     response = await client.request("GET", "https://example.com")
-        >>>     print(response.status_code)
-        200
-    """
-
-    def __init__(self, *args, **kwargs):
-        self._config = kwargs.pop("_config")
-
-        http_debug = self._config.get("http_debug", None)
-        if http_debug:
-            log.trace(f"Creating AsyncClient: {args}, {kwargs}")
-
-        self._persist_cookies = kwargs.pop("persist_cookies", True)
-
-        # timeout
-        http_timeout = self._config.get("http_timeout", 20)
-        if not "timeout" in kwargs:
-            kwargs["timeout"] = http_timeout
-
-        # headers
-        headers = kwargs.get("headers", None)
-        if headers is None:
-            headers = {}
-        # user agent
-        user_agent = self._config.get("user_agent", "BBOT")
-        if "User-Agent" not in headers:
-            headers["User-Agent"] = user_agent
-        kwargs["headers"] = headers
-        # proxy
-        proxies = self._config.get("http_proxy", None)
-        kwargs["proxies"] = proxies
-
-        super().__init__(*args, **kwargs)
-        if not self._persist_cookies:
-            self._cookies = DummyCookies()
-
-    def build_request(self, *args, **kwargs):
-        request = super().build_request(*args, **kwargs)
-        # add custom headers if the URL is in-scope
-        # TODO: re-enable this
-        # if self._preset.in_scope(str(request.url)):
-        #     for hk, hv in self._config.get("http_headers", {}).items():
-        #         # don't clobber headers
-        #         if hk not in request.headers:
-        #             request.headers[hk] = hv
-        return request
-
-    def _merge_cookies(self, cookies):
-        if self._persist_cookies:
-            return super()._merge_cookies(cookies)
-        return cookies
-
-
 class HTTPEngine(EngineServer):
 
     CMDS = {
@@ -92,7 +21,7 @@ class HTTPEngine(EngineServer):
         1: "request_batch",
         2: "request_custom_batch",
         3: "download",
-        99: "_mock",
+        99: "mock",
     }
 
     client_only_options = (
@@ -100,9 +29,9 @@ class HTTPEngine(EngineServer):
         "max_redirects",
     )
 
-    def __init__(self, socket_path, config={}):
+    def __init__(self, socket_path, target, config={}):
         super().__init__(socket_path)
-        self.log.critical("doing")
+        self.target = target
         self.config = config
         self.http_debug = self.config.get("http_debug", False)
         self._ssl_context_noverify = None
@@ -113,13 +42,13 @@ def __init__(self, socket_path, config={}):
 
     def AsyncClient(self, *args, **kwargs):
         kwargs["_config"] = self.config
+        kwargs["_target"] = self.target
         retries = kwargs.pop("retries", self.config.get("http_retries", 1))
         kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=self.ssl_verify)
         kwargs["verify"] = self.ssl_verify
         return BBOTAsyncClient(*args, **kwargs)
 
     async def request(self, *args, **kwargs):
-        self.log.critical(f"SERVER {args} / {kwargs}")
         raise_error = kwargs.pop("raise_error", False)
         # TODO: use this
         cache_for = kwargs.pop("cache_for", None)  # noqa
@@ -295,3 +224,11 @@ async def _acatch(self, url, raise_error):
                 log.trace(f"Unhandled exception with request to URL: {url}: {e}")
                 log.trace(traceback.format_exc())
             raise
+
+    def mock(self, *args, **kwargs):
+        import pytest
+        from bbot.core.helpers.interactsh import server_list as interactsh_servers
+
+        @pytest.fixture
+        def non_mocked_hosts() -> list:
+            return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 48ff47aaf..179abdc35 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -49,7 +49,8 @@ class WebHelper(EngineClient):
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.config = self.parent_helper.config
-        super().__init__(server_kwargs={"config": self.config})
+        self.ssl_verify = self.config.get("ssl_verify", False)
+        super().__init__(server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.radix_only})
 
     async def request(self, *args, **kwargs):
         """
@@ -466,48 +467,47 @@ def beautifulsoup(
             log.debug(f"Error parsing beautifulsoup: {e}")
             return False
 
-user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
-pass_keywords = [re.compile(r, re.I) for r in ["pass"]]
+    user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
+    pass_keywords = [re.compile(r, re.I) for r in ["pass"]]
 
+    def is_login_page(self, html):
+        """
+        Determines if the provided HTML content contains a login page.
 
-def is_login_page(html):
-    """
-    Determines if the provided HTML content contains a login page.
-
-    This function parses the HTML to search for forms with input fields typically used for
-    authentication. If it identifies password fields or a combination of username and password
-    fields, it returns True.
+        This function parses the HTML to search for forms with input fields typically used for
+        authentication. If it identifies password fields or a combination of username and password
+        fields, it returns True.
 
-    Args:
-        html (str): The HTML content to analyze.
+        Args:
+            html (str): The HTML content to analyze.
 
-    Returns:
-        bool: True if the HTML contains a login page, otherwise False.
+        Returns:
+            bool: True if the HTML contains a login page, otherwise False.
 
-    Examples:
-        >>> is_login_page('<form><input type="text" name="username"><input type="password" name="password"></form>')
-        True
+        Examples:
+            >>> is_login_page('<form><input type="text" name="username"><input type="password" name="password"></form>')
+            True
 
-        >>> is_login_page('<form><input type="text" name="search"></form>')
-        False
-    """
-    try:
-        soup = BeautifulSoup(html, "html.parser")
-    except Exception as e:
-        log.debug(f"Error parsing html: {e}")
-        return False
+            >>> is_login_page('<form><input type="text" name="search"></form>')
+            False
+        """
+        try:
+            soup = BeautifulSoup(html, "html.parser")
+        except Exception as e:
+            log.debug(f"Error parsing html: {e}")
+            return False
 
-    forms = soup.find_all("form")
+        forms = soup.find_all("form")
 
-    # first, check for obvious password fields
-    for form in forms:
-        if form.find_all("input", {"type": "password"}):
-            return True
+        # first, check for obvious password fields
+        for form in forms:
+            if form.find_all("input", {"type": "password"}):
+                return True
 
-    # next, check for forms that have both a user-like and password-like field
-    for form in forms:
-        user_fields = sum(bool(form.find_all("input", {"name": r})) for r in user_keywords)
-        pass_fields = sum(bool(form.find_all("input", {"name": r})) for r in pass_keywords)
-        if user_fields and pass_fields:
-            return True
-    return False
+        # next, check for forms that have both a user-like and password-like field
+        for form in forms:
+            user_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.user_keywords)
+            pass_fields = sum(bool(form.find_all("input", {"name": r})) for r in self.pass_keywords)
+            if user_fields and pass_fields:
+                return True
+        return False
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index c5e1a42f3..8b30ab39b 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -4,7 +4,6 @@
 import subprocess
 from pathlib import Path
 from bbot.modules.base import BaseModule
-from bbot.core.helpers.web import is_login_page
 
 
 class httpx(BaseModule):
@@ -163,7 +162,7 @@ async def handle_batch(self, *events):
             if httpx_ip:
                 tags.append(f"ip-{httpx_ip}")
             # detect login pages
-            if is_login_page(j.get("body", "")):
+            if self.helpers.web.is_login_page(j.get("body", "")):
                 tags.append("login-page")
             # grab title
             title = self.helpers.tagify(j.get("title", ""), maxlen=30)
diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 9c8f12fbe..f2254c519 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -11,7 +11,6 @@
 
 from bbot.errors import *
 from bbot.core import CORE
-from bbot.core.event.base import make_event
 from bbot.core.helpers.misc import make_table, mkdir, get_closest_match
 
 
@@ -234,19 +233,11 @@ def __init__(
         self._module_dirs = set()
         self.module_dirs = module_dirs
 
-        self.strict_scope = strict_scope
-
         # target / whitelist / blacklist
-        from bbot.scanner.target import Target
+        from bbot.scanner.target import BBOTTarget
 
-        self.target = Target(*targets, strict_scope=self.strict_scope)
-        if not whitelist:
-            self.whitelist = self.target.copy()
-        else:
-            self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
-        if not blacklist:
-            blacklist = []
-        self.blacklist = Target(*blacklist)
+        self.strict_scope = strict_scope
+        self.target = BBOTTarget(targets, whitelist=whitelist, blacklist=blacklist, strict_scope=self.strict_scope)
 
         # include other presets
         if include and not isinstance(include, (list, tuple, set)):
@@ -267,6 +258,14 @@ def __init__(
     def bbot_home(self):
         return Path(self.config.get("home", "~/.bbot")).expanduser().resolve()
 
+    @property
+    def whitelist(self):
+        return self.target.whitelist
+
+    @property
+    def blacklist(self):
+        return self.target.blacklist
+
     @property
     def preset_dir(self):
         return self.bbot_home / "presets"
@@ -325,11 +324,11 @@ def merge(self, other):
         self.explicit_output_modules.update(other.explicit_output_modules)
         self.flags.update(other.flags)
         # scope
-        self.target.add(other.target)
-        self.whitelist.add(other.whitelist)
-        self.blacklist.add(other.blacklist)
+        self.target.target.add(other.target.target)
+        self.target.whitelist.add(other.target.whitelist)
+        self.target.blacklist.add(other.target.blacklist)
         self.strict_scope = self.strict_scope or other.strict_scope
-        for t in (self.target, self.whitelist):
+        for t in (self.target.target, self.target.whitelist):
             t.strict_scope = self.strict_scope
         # log verbosity
         if other.silent:
@@ -565,58 +564,13 @@ def args(self):
         return self._args
 
     def in_scope(self, host):
-        """
-        Check whether a hostname, url, IP, etc. is in scope.
-        Accepts either events or string data.
-
-        Checks whitelist and blacklist.
-        If `host` is an event and its scope distance is zero, it will automatically be considered in-scope.
-
-        Examples:
-            Check if a URL is in scope:
-            >>> preset.in_scope("http://www.evilcorp.com")
-            True
-        """
-        try:
-            e = make_event(host, dummy=True)
-        except ValidationError:
-            return False
-        in_scope = e.scope_distance == 0 or self.whitelisted(e)
-        return in_scope and not self.blacklisted(e)
+        return self.target.in_scope(host)
 
     def blacklisted(self, host):
-        """
-        Check whether a hostname, url, IP, etc. is blacklisted.
-
-        Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute.
-
-        Args:
-            host (str or IPAddress or Event): The host to check against the blacklist
-
-        Examples:
-            Check if a URL's host is blacklisted:
-            >>> preset.blacklisted("http://www.evilcorp.com")
-            True
-        """
-        e = make_event(host, dummy=True)
-        return e in self.blacklist
+        return self.target.blacklisted(host)
 
     def whitelisted(self, host):
-        """
-        Check whether a hostname, url, IP, etc. is whitelisted.
-
-        Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute.
-
-        Args:
-            host (str or IPAddress or Event): The host to check against the whitelist
-
-        Examples:
-            Check if a URL's host is whitelisted:
-            >>> preset.whitelisted("http://www.evilcorp.com")
-            True
-        """
-        e = make_event(host, dummy=True)
-        return e in self.whitelist
+        return self.target.whitelisted(host)
 
     @classmethod
     def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
@@ -734,8 +688,8 @@ def to_dict(self, include_target=False, full_config=False):
         # scope
         if include_target:
             target = sorted(str(t.data) for t in self.target)
-            whitelist = sorted(str(t.data) for t in self.whitelist)
-            blacklist = sorted(str(t.data) for t in self.blacklist)
+            whitelist = sorted(str(t.data) for t in self.target.whitelist)
+            blacklist = sorted(str(t.data) for t in self.target.blacklist)
             if target:
                 preset_dict["target"] = target
             if whitelist and whitelist != target:
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index 9cca55ecc..ed0fc6066 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -13,6 +13,109 @@
 log = logging.getLogger("bbot.core.target")
 
 
+class BBOTTarget:
+    """
+    A convenient abstraction of a scan target that includes whitelisting and blacklisting
+
+    Provides high-level functions like in_scope(), which includes both whitelist and blacklist checks.
+    """
+
+    def __init__(self, targets, whitelist=None, blacklist=None, strict_scope=False):
+        self.strict_scope = strict_scope
+        self.target = Target(*targets, strict_scope=self.strict_scope)
+        if whitelist is None:
+            self.whitelist = self.target.copy()
+        else:
+            self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
+        if blacklist is None:
+            blacklist = []
+        self.blacklist = Target(*blacklist)
+
+    def __iter__(self):
+        return iter(self.target)
+
+    def __len__(self):
+        return len(self.target)
+
+    def __contains__(self, other):
+        return other in self.target
+
+    @property
+    def events(self):
+        return self.target.events
+
+    def in_scope(self, host):
+        """
+        Check whether a hostname, url, IP, etc. is in scope.
+        Accepts either events or string data.
+
+        Checks whitelist and blacklist.
+        If `host` is an event and its scope distance is zero, it will automatically be considered in-scope.
+
+        Examples:
+            Check if a URL is in scope:
+            >>> preset.in_scope("http://www.evilcorp.com")
+            True
+        """
+        try:
+            e = make_event(host, dummy=True)
+        except ValidationError:
+            return False
+        in_scope = e.scope_distance == 0 or self.whitelisted(e)
+        return in_scope and not self.blacklisted(e)
+
+    def blacklisted(self, host):
+        """
+        Check whether a hostname, url, IP, etc. is blacklisted.
+
+        Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute.
+
+        Args:
+            host (str or IPAddress or Event): The host to check against the blacklist
+
+        Examples:
+            Check if a URL's host is blacklisted:
+            >>> preset.blacklisted("http://www.evilcorp.com")
+            True
+        """
+        e = make_event(host, dummy=True)
+        return e in self.blacklist
+
+    def whitelisted(self, host):
+        """
+        Check whether a hostname, url, IP, etc. is whitelisted.
+
+        Note that `host` can be a hostname, IP address, CIDR, email address, or any BBOT `Event` with the `host` attribute.
+
+        Args:
+            host (str or IPAddress or Event): The host to check against the whitelist
+
+        Examples:
+            Check if a URL's host is whitelisted:
+            >>> preset.whitelisted("http://www.evilcorp.com")
+            True
+        """
+        e = make_event(host, dummy=True)
+        return e in self.whitelist
+
+    @property
+    def radix_only(self):
+        """
+        A slimmer, serializable version of the target designed for simple scope checks
+        """
+        return self.__class__(
+            targets=[e.host for e in self.target if e.host],
+            whitelist=[e.host for e in self.whitelist if e.host],
+            blacklist=[e.host for e in self.blacklist if e.host],
+            strict_scope=self.strict_scope,
+        )
+        # class RadixOnlyTarget:
+        #     def __init__(self, targets, whitelist, blacklist):
+        #         self.target = Target(*targets, strict_scope=self.strict_scope)
+        #         self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
+        #         self.blacklist = Target(*blacklist)
+
+
 class Target:
     """
     A class representing a target. Can contain an unlimited number of hosts, IP or IP ranges, URLs, etc.
@@ -118,13 +221,14 @@ def add(self, t, event_type=None):
         if not isinstance(t, (list, tuple, set)):
             t = [t]
         for single_target in t:
-            if type(single_target) == self.__class__:
+            if isinstance(single_target, self.__class__):
                 for event in single_target.events:
                     self._add_event(event)
             else:
                 if is_event(single_target):
                     event = single_target
                 else:
+                    single_target = str(single_target)
                     for eventtype, regex in self.special_event_types.items():
                         match = regex.match(single_target)
                         if match:
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 3dd403106..058baafc0 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -10,7 +10,6 @@
 
 from bbot.core import CORE
 from bbot.core.helpers.misc import execute_sync_or_async
-from bbot.core.helpers.interactsh import server_list as interactsh_servers
 
 
 test_config = OmegaConf.load(Path(__file__).parent / "test.conf")
@@ -43,11 +42,6 @@ def pytest_sessionfinish(session, exitstatus):
     yield
 
 
-@pytest.fixture
-def non_mocked_hosts() -> list:
-    return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
-
-
 @pytest.fixture
 def assert_all_responses_were_requested() -> bool:
     return False
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index aeac2ba2f..4c3b69c7b 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -298,11 +298,13 @@ async def test_http_ssl(bbot_scanner, bbot_httpserver_ssl):
 @pytest.mark.asyncio
 async def test_web_cookies(bbot_scanner, httpx_mock):
     import httpx
+    from bbot.core.helpers.web.client import BBOTAsyncClient
 
     # make sure cookies work when enabled
     httpx_mock.add_response(url="http://www.evilcorp.com/cookies", headers=[("set-cookie", "wat=asdf; path=/")])
     scan = bbot_scanner()
-    client = scan.helpers.AsyncClient(persist_cookies=True)
+
+    client = BBOTAsyncClient(persist_cookies=True, _config=scan.config, _target=scan.target)
     r = await client.get(url="http://www.evilcorp.com/cookies")
     assert r.cookies["wat"] == "asdf"
     httpx_mock.add_response(url="http://www.evilcorp.com/cookies/test", match_headers={"Cookie": "wat=asdf"})
@@ -315,7 +317,7 @@ async def test_web_cookies(bbot_scanner, httpx_mock):
     # make sure they don't when they're not
     httpx_mock.add_response(url="http://www2.evilcorp.com/cookies", headers=[("set-cookie", "wats=fdsa; path=/")])
     scan = bbot_scanner()
-    client2 = scan.helpers.AsyncClient(persist_cookies=False)
+    client2 = BBOTAsyncClient(persist_cookies=False, _config=scan.config, _target=scan.target)
     r = await client2.get(url="http://www2.evilcorp.com/cookies")
     # make sure we can access the cookies
     assert "wats" in r.cookies

From 0898fbf9e4e9ca1603d943363509179877da883f Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 7 May 2024 18:03:05 -0400
Subject: [PATCH 084/220] adding badsecrets custom file initial

---
 bbot/modules/badsecrets.py                    | 19 +++++++++++++--
 .../module_tests/test_module_badsecrets.py    | 23 +++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index 01cc36ed8..d7a475df3 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -1,7 +1,6 @@
 import multiprocessing
-
+from pathlib import Path
 from .base import BaseModule
-
 from badsecrets.base import carve_all_modules
 
 
@@ -10,8 +9,23 @@ class badsecrets(BaseModule):
     produced_events = ["FINDING", "VULNERABILITY", "TECHNOLOGY"]
     flags = ["active", "safe", "web-basic"]
     meta = {"description": "Library for detecting known or weak secrets across many web frameworks"}
+    options = {"custom_secrets": None}
+    options_desc = {
+        "custom_secrets": "Include custom secrets loaded from a local file",
+    }
     deps_pip = ["badsecrets~=0.4.490"]
 
+    async def setup(self):
+        self.custom_secrets = None
+        custom_secrets = self.config.get("custom_secrets", None)
+        if custom_secrets:
+            if Path(custom_secrets).is_file():
+                self.custom_secrets = custom_secrets
+            else:
+                self.warning(f"custom secrets file [{custom_secrets}] is not valid")
+                return None, "Custom secrets file not valid"
+        return True
+
     @property
     def _max_event_handlers(self):
         return max(1, multiprocessing.cpu_count() - 1)
@@ -39,6 +53,7 @@ async def handle_event(self, event):
                     headers=resp_headers,
                     cookies=resp_cookies,
                     url=event.data.get("url", None),
+                    custom_resource=self.custom_secrets,
                 )
             except Exception as e:
                 self.warning(f"Error processing {event}: {e}")
diff --git a/bbot/test/test_step_2/module_tests/test_module_badsecrets.py b/bbot/test/test_step_2/module_tests/test_module_badsecrets.py
index 23063f2dc..8ea42031b 100644
--- a/bbot/test/test_step_2/module_tests/test_module_badsecrets.py
+++ b/bbot/test/test_step_2/module_tests/test_module_badsecrets.py
@@ -121,3 +121,26 @@ def check(self, module_test, events):
         assert CookieBasedDetection, "No JWT cookie vuln detected"
         assert CookieBasedDetection_2, "No Express.js cookie vuln detected"
         assert CookieBasedDetection_3, "No Express.js (cs dual cookies) vuln detected"
+
+
+# class TestBadSecrets_customsedrets(TestBadSecrets):
+
+
+#     sample_viewstate = """
+#     <form method="post" action="./query.aspx" id="form1">
+# <div class="aspNetHidden">
+# <input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="rJdyYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESf4nBu" />
+# </div>
+
+# <div class="aspNetHidden">
+
+#     <input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="EDD8C9AE" />
+#     <input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
+# </div>
+#     </form>
+# </body>
+# </html>
+# """
+
+
+# # config_overrides = {"custom_secrets": TEMP FILE}
\ No newline at end of file

From 371561b9f885a95b5bf6687699361c24389e5ee9 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 7 May 2024 18:51:43 -0400
Subject: [PATCH 085/220] badsecrets custom secrets tests

---
 bbot/modules/badsecrets.py                    |  1 +
 .../module_tests/test_module_badsecrets.py    | 54 ++++++++++++-------
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index d7a475df3..0f9eca7dd 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -21,6 +21,7 @@ async def setup(self):
         if custom_secrets:
             if Path(custom_secrets).is_file():
                 self.custom_secrets = custom_secrets
+                self.info(f"Successfully loaded secrets file [{custom_secrets}]")
             else:
                 self.warning(f"custom secrets file [{custom_secrets}] is not valid")
                 return None, "Custom secrets file not valid"
diff --git a/bbot/test/test_step_2/module_tests/test_module_badsecrets.py b/bbot/test/test_step_2/module_tests/test_module_badsecrets.py
index 8ea42031b..9eda654eb 100644
--- a/bbot/test/test_step_2/module_tests/test_module_badsecrets.py
+++ b/bbot/test/test_step_2/module_tests/test_module_badsecrets.py
@@ -1,4 +1,4 @@
-from .base import ModuleTestBase
+from .base import ModuleTestBase, tempwordlist
 
 
 class TestBadSecrets(ModuleTestBase):
@@ -123,24 +123,42 @@ def check(self, module_test, events):
         assert CookieBasedDetection_3, "No Express.js (cs dual cookies) vuln detected"
 
 
-# class TestBadSecrets_customsedrets(TestBadSecrets):
-
-
-#     sample_viewstate = """
-#     <form method="post" action="./query.aspx" id="form1">
-# <div class="aspNetHidden">
-# <input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="rJdyYspajyiWEjvZ/SMXsU/1Q6Dp1XZ/19fZCABpGqWu+s7F1F/JT1s9mP9ED44fMkninhDc8eIq7IzSllZeJ9JVUME41i8ozheGunVSaESf4nBu" />
-# </div>
+class TestBadSecrets_customsecrets(TestBadSecrets):
+    config_overrides = {
+        "modules": {
+            "badsecrets": {
+                "custom_secrets": tempwordlist(
+                    [
+                        "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF,DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+                    ]
+                )
+            }
+        }
+    }
 
-# <div class="aspNetHidden">
+    sample_viewstate = """
+    <form method="post" action="./query.aspx" id="form1">
+<div class="aspNetHidden">
+<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE" value="/wEPDwUJODExMDE5NzY5ZGS02CHaDxi5Kw19mPShbrrOUCJ4pA==" />
+</div>
 
-#     <input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="EDD8C9AE" />
-#     <input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
-# </div>
-#     </form>
-# </body>
-# </html>
-# """
+<div class="aspNetHidden">
 
+    <input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="75BBA7D6" />
+    <input type="hidden" name="__VIEWSTATEENCRYPTED" id="__VIEWSTATEENCRYPTED" value="" />
+</div>
+    </form>
+</body>
+</html>
+"""
 
-# # config_overrides = {"custom_secrets": TEMP FILE}
\ No newline at end of file
+    def check(self, module_test, events):
+        SecretFound = False
+        for e in events:
+            if (
+                e.type == "VULNERABILITY"
+                and "Known Secret Found." in e.data["description"]
+                and "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" in e.data["description"]
+            ):
+                SecretFound = True
+        assert SecretFound, "No secret found"

From 07d38bc78792add5000be6568adb4d8bd851ec95 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 8 May 2024 10:09:56 -0400
Subject: [PATCH 086/220] add client.py

---
 bbot/core/helpers/web/client.py | 77 +++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 bbot/core/helpers/web/client.py

diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py
new file mode 100644
index 000000000..32cf47067
--- /dev/null
+++ b/bbot/core/helpers/web/client.py
@@ -0,0 +1,77 @@
+import httpx
+import logging
+from httpx._models import Cookies
+
+log = logging.getLogger("bbot.core.helpers.web.client")
+
+
+class DummyCookies(Cookies):
+    def extract_cookies(self, *args, **kwargs):
+        pass
+
+
+class BBOTAsyncClient(httpx.AsyncClient):
+    """
+    A subclass of httpx.AsyncClient tailored with BBOT-specific configurations and functionalities.
+    This class provides rate limiting, logging, configurable timeouts, user-agent customization, custom
+    headers, and proxy settings. Additionally, it allows the disabling of cookies, making it suitable
+    for use across an entire scan.
+
+    Attributes:
+        _bbot_scan (object): BBOT scan object containing configuration details.
+        _persist_cookies (bool): Flag to determine whether cookies should be persisted across requests.
+
+    Examples:
+        >>> async with BBOTAsyncClient(_bbot_scan=bbot_scan_object) as client:
+        >>>     response = await client.request("GET", "https://example.com")
+        >>>     print(response.status_code)
+        200
+    """
+
+    def __init__(self, *args, **kwargs):
+        self._config = kwargs.pop("_config")
+        self._target = kwargs.pop("_target")
+
+        http_debug = self._config.get("http_debug", None)
+        if http_debug:
+            log.trace(f"Creating AsyncClient: {args}, {kwargs}")
+
+        self._persist_cookies = kwargs.pop("persist_cookies", True)
+
+        # timeout
+        http_timeout = self._config.get("http_timeout", 20)
+        if not "timeout" in kwargs:
+            kwargs["timeout"] = http_timeout
+
+        # headers
+        headers = kwargs.get("headers", None)
+        if headers is None:
+            headers = {}
+        # user agent
+        user_agent = self._config.get("user_agent", "BBOT")
+        if "User-Agent" not in headers:
+            headers["User-Agent"] = user_agent
+        kwargs["headers"] = headers
+        # proxy
+        proxies = self._config.get("http_proxy", None)
+        kwargs["proxies"] = proxies
+
+        super().__init__(*args, **kwargs)
+        if not self._persist_cookies:
+            self._cookies = DummyCookies()
+
+    def build_request(self, *args, **kwargs):
+        request = super().build_request(*args, **kwargs)
+        # add custom headers if the URL is in-scope
+        # TODO: re-enable this
+        if self._target.in_scope(str(request.url)):
+            for hk, hv in self._config.get("http_headers", {}).items():
+                # don't clobber headers
+                if hk not in request.headers:
+                    request.headers[hk] = hv
+        return request
+
+    def _merge_cookies(self, cookies):
+        if self._persist_cookies:
+            return super()._merge_cookies(cookies)
+        return cookies

From b429299501d31a084d7ef7bb96b429a1e815c8a1 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 8 May 2024 11:51:16 -0400
Subject: [PATCH 087/220] include host in event json

---
 bbot/core/event/base.py              | 18 +++++++++++++++---
 bbot/test/test_step_1/test_events.py | 25 ++++++++++++++++---------
 2 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ac9ca654d..66116d87d 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -608,11 +608,13 @@ def json(self, mode="json", siem_friendly=False):
         Returns:
             dict: JSON-serializable dictionary representation of the event object.
         """
+        # type, ID, scope description
         j = dict()
         for i in ("type", "id", "scope_description"):
             v = getattr(self, i, "")
             if v:
                 j.update({i: v})
+        # event data
         data_attr = getattr(self, f"data_{mode}", None)
         if data_attr is not None:
             data = data_attr
@@ -622,23 +624,33 @@ def json(self, mode="json", siem_friendly=False):
             j["data"] = {self.type: data}
         else:
             j["data"] = data
+        # host, dns children
+        if self.host:
+            j["host"] = str(self.host)
+            j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
+            j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        # web spider distance
         web_spider_distance = getattr(self, "web_spider_distance", None)
         if web_spider_distance is not None:
             j["web_spider_distance"] = web_spider_distance
+        # scope distance
         j["scope_distance"] = self.scope_distance
+        # scan
         if self.scan:
             j["scan"] = self.scan.id
+        # timestamp
         j["timestamp"] = self.timestamp.timestamp()
-        if self.host:
-            j["resolved_hosts"] = sorted(str(h) for h in self.resolved_hosts)
-            j["dns_children"] = {k: list(v) for k, v in self.dns_children.items()}
+        # parent event
         source_id = self.source_id
         if source_id:
             j["source"] = source_id
+        # tags
         if self.tags:
             j.update({"tags": list(self.tags)})
+        # source module
         if self.module:
             j.update({"module": str(self.module)})
+        # sequence of modules that led to discovery
         if self.module_sequence:
             j.update({"module_sequence": str(self.module_sequence)})
 
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index f831391be..103c72946 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -399,33 +399,40 @@ async def test_events(events, helpers):
     # test event serialization
     from bbot.core.event import event_from_json
 
-    db_event = scan.make_event("evilcorp.com", dummy=True)
+    db_event = scan.make_event("evilcorp.com:80", dummy=True)
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
     timestamp = db_event.timestamp.timestamp()
     json_event = db_event.json()
     assert json_event["scope_distance"] == 1
-    assert json_event["data"] == "evilcorp.com"
-    assert json_event["type"] == "DNS_NAME"
+    assert json_event["data"] == "evilcorp.com:80"
+    assert json_event["type"] == "OPEN_TCP_PORT"
+    assert json_event["host"] == "evilcorp.com"
     assert json_event["timestamp"] == timestamp
     reconstituted_event = event_from_json(json_event)
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.timestamp() == timestamp
-    assert reconstituted_event.data == "evilcorp.com"
-    assert reconstituted_event.type == "DNS_NAME"
+    assert reconstituted_event.data == "evilcorp.com:80"
+    assert reconstituted_event.type == "OPEN_TCP_PORT"
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
+    hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
+    hostless_event_json = hostless_event.json()
+    assert hostless_event_json["type"] == "ASDF"
+    assert hostless_event_json["data"] == "asdf"
+    assert not "host" in hostless_event_json
 
     # SIEM-friendly serialize/deserialize
     json_event_siemfriendly = db_event.json(siem_friendly=True)
     assert json_event_siemfriendly["scope_distance"] == 1
-    assert json_event_siemfriendly["data"] == {"DNS_NAME": "evilcorp.com"}
-    assert json_event_siemfriendly["type"] == "DNS_NAME"
+    assert json_event_siemfriendly["data"] == {"OPEN_TCP_PORT": "evilcorp.com:80"}
+    assert json_event_siemfriendly["type"] == "OPEN_TCP_PORT"
+    assert json_event_siemfriendly["host"] == "evilcorp.com"
     assert json_event_siemfriendly["timestamp"] == timestamp
     reconstituted_event2 = event_from_json(json_event_siemfriendly, siem_friendly=True)
     assert reconstituted_event2.scope_distance == 1
     assert reconstituted_event2.timestamp.timestamp() == timestamp
-    assert reconstituted_event2.data == "evilcorp.com"
-    assert reconstituted_event2.type == "DNS_NAME"
+    assert reconstituted_event2.data == "evilcorp.com:80"
+    assert reconstituted_event2.type == "OPEN_TCP_PORT"
     assert "127.0.0.1" in reconstituted_event2.resolved_hosts
 
     http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event)

From a8b1318316fe76992356fb9647660c36ae17bcc2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 8 May 2024 12:53:28 -0400
Subject: [PATCH 088/220] more host tests

---
 bbot/test/test_step_1/test_events.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 103c72946..530969a00 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -414,6 +414,7 @@ async def test_events(events, helpers):
     assert reconstituted_event.timestamp.timestamp() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
     assert reconstituted_event.type == "OPEN_TCP_PORT"
+    assert reconstituted_event.host == "evilcorp.com"
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()
@@ -433,6 +434,7 @@ async def test_events(events, helpers):
     assert reconstituted_event2.timestamp.timestamp() == timestamp
     assert reconstituted_event2.data == "evilcorp.com:80"
     assert reconstituted_event2.type == "OPEN_TCP_PORT"
+    assert reconstituted_event2.host == "evilcorp.com"
     assert "127.0.0.1" in reconstituted_event2.resolved_hosts
 
     http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event)
@@ -443,10 +445,12 @@ async def test_events(events, helpers):
     json_event = http_response.json()
     assert isinstance(json_event["data"], dict)
     assert json_event["type"] == "HTTP_RESPONSE"
+    assert json_event["host"] == "example.com"
     assert json_event["source"] == scan.root_event.id
     reconstituted_event = event_from_json(json_event)
     assert isinstance(reconstituted_event.data, dict)
     assert reconstituted_event.data["input"] == "http://example.com:80"
+    assert reconstituted_event.host == "example.com"
     assert reconstituted_event.type == "HTTP_RESPONSE"
     assert reconstituted_event.source_id == scan.root_event.id
 

From ac345ccf5543bdfa6fef8f3d18605db40b6c049d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 8 May 2024 17:58:39 -0400
Subject: [PATCH 089/220] engine tests, async cancellation

---
 bbot/core/engine.py                  | 73 ++++++++++++++++++++--------
 bbot/core/helpers/web/engine.py      | 17 ++-----
 bbot/modules/ntlm.py                 | 42 ++++++----------
 bbot/test/conftest.py                |  6 +++
 bbot/test/test_step_1/test_engine.py | 58 ++++++++++++++++++++++
 bbot/test/test_step_1/test_web.py    | 20 ++++++--
 6 files changed, 152 insertions(+), 64 deletions(-)
 create mode 100644 bbot/test/test_step_1/test_engine.py

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 9242630c2..2e08ca894 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -46,9 +46,6 @@ def check_error(self, message):
         if message is error_sentinel:
             return True
         if isinstance(message, dict) and len(message) == 1 and "_e" in message:
-            error, trace = message["_e"]
-            self.log.error(error)
-            self.log.trace(trace)
             return True
         return False
 
@@ -76,11 +73,17 @@ def __init__(self, **kwargs):
 
     async def run_and_return(self, command, *args, **kwargs):
         async with self.new_socket() as socket:
-            message = self.make_message(command, args=args, kwargs=kwargs)
-            if message is error_sentinel:
-                return
-            await socket.send(message)
-            binary = await socket.recv()
+            try:
+                message = self.make_message(command, args=args, kwargs=kwargs)
+                if message is error_sentinel:
+                    return
+                await socket.send(message)
+                binary = await socket.recv()
+            except BaseException:
+                # -1 == special "cancel" signal
+                cancel_message = pickle.dumps({"c": -1})
+                await socket.send(cancel_message)
+                raise
         # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}")
         message = self.unpickle(binary)
         self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}")
@@ -96,14 +99,20 @@ async def run_and_yield(self, command, *args, **kwargs):
         async with self.new_socket() as socket:
             await socket.send(message)
             while 1:
-                binary = await socket.recv()
-                # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}")
-                message = self.unpickle(binary)
-                self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}")
-                # error handling
-                if self.check_error(message) or self.check_stop(message):
-                    break
-                yield message
+                try:
+                    binary = await socket.recv()
+                    # self.log.debug(f"{self.name}.{command}({kwargs}) got binary: {binary}")
+                    message = self.unpickle(binary)
+                    self.log.debug(f"{self.name}.{command}({kwargs}) got message: {message}")
+                    # error handling
+                    if self.check_error(message) or self.check_stop(message):
+                        break
+                    yield message
+                except GeneratorExit:
+                    # -1 == special "cancel" signal
+                    cancel_message = pickle.dumps({"c": -1})
+                    await socket.send(cancel_message)
+                    raise
 
     def check_stop(self, message):
         if isinstance(message, dict) and len(message) == 1 and "_s" in message:
@@ -188,15 +197,21 @@ def __init__(self, socket_path):
             self.socket = self.context.socket(zmq.ROUTER)
             # create socket file
             self.socket.bind(f"ipc://{socket_path}")
+            # task <--> client id mapping
+            self.tasks = dict()
 
     async def run_and_return(self, client_id, command_fn, *args, **kwargs):
         self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({kwargs})")
         try:
             result = await command_fn(*args, **kwargs)
-        except Exception as e:
+        except ValueError as e:
             error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
             trace = traceback.format_exc()
+            self.log.error(error)
+            self.log.trace(trace)
             result = {"_e": (error, trace)}
+        finally:
+            self.tasks.pop(client_id, None)
         await self.send_socket_multipart([client_id, pickle.dumps(result)])
 
     async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
@@ -205,11 +220,15 @@ async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
             async for _ in command_fn(*args, **kwargs):
                 await self.send_socket_multipart([client_id, pickle.dumps(_)])
             await self.send_socket_multipart([client_id, pickle.dumps({"_s": None})])
-        except Exception as e:
+        except ValueError as e:
             error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
             trace = traceback.format_exc()
+            self.log.error(error)
+            self.log.trace(trace)
             result = {"_e": (error, trace)}
             await self.send_socket_multipart([client_id, pickle.dumps(result)])
+        finally:
+            self.tasks.pop(client_id, None)
 
     async def send_socket_multipart(self, *args, **kwargs):
         try:
@@ -232,6 +251,21 @@ async def worker(self):
                     self.log.warning(f"No command sent in message: {message}")
                     continue
 
+                if cmd == -1:
+                    task, _cmd, _args, _kwargs = self.tasks.get(client_id, None)
+                    if task is not None:
+                        self.log.debug(f"Cancelling client id {client_id} (task: {task})")
+                        task.cancel()
+                        try:
+                            await task
+                        except (KeyboardInterrupt, asyncio.CancelledError):
+                            pass
+                        except BaseException as e:
+                            self.log.error(f"Unhandled error in {_cmd}({_args}, {_kwargs}): {e}")
+                            self.log.trace(traceback.format_exc())
+                        self.tasks.pop(client_id, None)
+                    continue
+
                 args = message.get("a", ())
                 if not isinstance(args, tuple):
                     self.log.warning(f"{self.name}: received invalid args of type {type(args)}, should be tuple")
@@ -253,7 +287,8 @@ async def worker(self):
                 else:
                     coroutine = self.run_and_return(client_id, command_fn, *args, **kwargs)
 
-                asyncio.create_task(coroutine)
+                task = asyncio.create_task(coroutine)
+                self.tasks[client_id] = task, command_fn, args, kwargs
         except Exception as e:
             self.log.error(f"Error in EngineServer worker: {e}")
             self.log.trace(traceback.format_exc())
diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 7b9260f29..fd35a1f5d 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -21,7 +21,6 @@ class HTTPEngine(EngineServer):
         1: "request_batch",
         2: "request_custom_batch",
         3: "download",
-        99: "mock",
     }
 
     client_only_options = (
@@ -107,12 +106,10 @@ def new_task(url):
             done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
 
             for task in done:
-                results = task.result()
+                response = task.result()
                 url = tasks.pop(task)
-
-                if results:
-                    yield (url, results)
-
+                if response:
+                    yield (url, response)
                 if urls:  # Start a new task for each one completed, if URLs remain
                     new_task(urls.pop(0))
 
@@ -224,11 +221,3 @@ async def _acatch(self, url, raise_error):
                 log.trace(f"Unhandled exception with request to URL: {url}: {e}")
                 log.trace(traceback.format_exc())
             raise
-
-    def mock(self, *args, **kwargs):
-        import pytest
-        from bbot.core.helpers.interactsh import server_list as interactsh_servers
-
-        @pytest.fixture
-        def non_mocked_hosts() -> list:
-            return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index b08fe4784..66c7dab9d 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -76,15 +76,15 @@ class ntlm(BaseModule):
     in_scope_only = True
 
     async def setup(self):
-        self.processed = set()
         self.found = set()
         self.try_all = self.config.get("try_all", False)
         return True
 
     async def handle_event(self, event):
         found_hash = hash(f"{event.host}:{event.port}")
+        agen = self.handle_url(event)
         if found_hash not in self.found:
-            for result, request_url in await self.handle_url(event):
+            async for result, request_url in agen:
                 if result and request_url:
                     self.found.add(found_hash)
                     await self.emit_event(
@@ -99,6 +99,7 @@ async def handle_event(self, event):
                     fqdn = result.get("FQDN", "")
                     if fqdn:
                         await self.emit_event(fqdn, "DNS_NAME", source=event)
+                    await agen.aclose()
                     break
 
     async def filter_event(self, event):
@@ -124,28 +125,15 @@ async def handle_url(self, event):
             for endpoint in ntlm_discovery_endpoints:
                 urls.add(f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{endpoint}")
 
-        tasks = []
-        for url in urls:
-            url_hash = hash(url)
-            if url_hash in self.processed:
-                continue
-            self.processed.add(url_hash)
-            tasks.append(self.helpers.create_task(self.check_ntlm(url)))
-
-        return await self.helpers.gather(*tasks)
-
-    async def check_ntlm(self, test_url):
-        # use lower timeout value
-        http_timeout = self.config.get("httpx_timeout", 5)
-        r = await self.helpers.request(test_url, headers=NTLM_test_header, allow_redirects=False, timeout=http_timeout)
-        ntlm_resp = r.headers.get("WWW-Authenticate", "")
-        if ntlm_resp:
-            ntlm_resp_b64 = max(ntlm_resp.split(","), key=lambda x: len(x)).split()[-1]
-            try:
-                ntlm_resp_decoded = self.helpers.ntlm.ntlmdecode(ntlm_resp_b64)
-                if ntlm_resp_decoded:
-                    return ntlm_resp_decoded, test_url
-            except NTLMError as e:
-                self.verbose(str(e))
-                return None, test_url
-        return None, test_url
+        async for url, response in self.helpers.request_batch(
+            urls, headers=NTLM_test_header, allow_redirects=False, timeout=self.http_timeout
+        ):
+            ntlm_resp = response.headers.get("WWW-Authenticate", "")
+            if ntlm_resp:
+                ntlm_resp_b64 = max(ntlm_resp.split(","), key=lambda x: len(x)).split()[-1]
+                try:
+                    ntlm_resp_decoded = self.helpers.ntlm.ntlmdecode(ntlm_resp_b64)
+                    if ntlm_resp_decoded:
+                        yield ntlm_resp_decoded, url
+                except NTLMError as e:
+                    self.verbose(str(e))
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 058baafc0..9d0a8fab7 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -10,6 +10,7 @@
 
 from bbot.core import CORE
 from bbot.core.helpers.misc import execute_sync_or_async
+from bbot.core.helpers.interactsh import server_list as interactsh_servers
 
 
 test_config = OmegaConf.load(Path(__file__).parent / "test.conf")
@@ -88,6 +89,11 @@ def bbot_httpserver_ssl():
     server.clear()
 
 
+@pytest.fixture
+def non_mocked_hosts() -> list:
+    return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers
+
+
 @pytest.fixture
 def bbot_httpserver_allinterfaces():
     server = HTTPServer(host="0.0.0.0", port=5556)
diff --git a/bbot/test/test_step_1/test_engine.py b/bbot/test/test_step_1/test_engine.py
new file mode 100644
index 000000000..5413e3b03
--- /dev/null
+++ b/bbot/test/test_step_1/test_engine.py
@@ -0,0 +1,58 @@
+from ..bbot_fixtures import *
+
+
+@pytest.mark.asyncio
+async def test_engine():
+    from bbot.core.engine import EngineClient, EngineServer
+
+    counter = 0
+
+    class TestEngineServer(EngineServer):
+
+        CMDS = {
+            0: "return_thing",
+            1: "yield_stuff",
+        }
+
+        async def return_thing(self):
+            return "thing"
+
+        async def yield_stuff(self, n):
+            nonlocal counter
+            for i in range(n):
+                yield f"thing{i}"
+                counter += 1
+                await asyncio.sleep(0.1)
+
+    class TestEngineClient(EngineClient):
+
+        SERVER_CLASS = TestEngineServer
+
+        async def return_thing(self):
+            return await self.run_and_return("return_thing")
+
+        async def yield_stuff(self, n):
+            async for _ in self.run_and_yield("yield_stuff", n):
+                yield _
+
+    # test return functionality
+    test_engine = TestEngineClient()
+    return_res = await test_engine.return_thing()
+    assert return_res == "thing"
+
+    # test async generator
+    assert counter == 0
+    yield_res = [r async for r in test_engine.yield_stuff(13)]
+    assert yield_res == [f"thing{i}" for i in range(13)]
+    assert len(yield_res) == 13
+    assert counter == 13
+
+    # test async generator with cancellation
+    counter = 0
+    agen = test_engine.yield_stuff(1000)
+    async for r in agen:
+        if counter > 10:
+            await agen.aclose()
+            break
+    await asyncio.sleep(5)
+    assert counter < 15
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index 4c3b69c7b..2ccf06e32 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -6,26 +6,38 @@
 @pytest.mark.asyncio
 async def test_web_engine(bbot_scanner, bbot_httpserver):
 
+    web_body = "hello_there" * 1000
     url = bbot_httpserver.url_for("/test")
-    bbot_httpserver.expect_request(uri="/test").respond_with_data("hello_there")
+    bbot_httpserver.expect_request(uri="/test").respond_with_data(web_body)
 
     scan = bbot_scanner()
 
     # request
     response = await scan.helpers.request(url)
     assert response.status_code > 0
-    assert response.text == "hello_there"
+    assert response.text == web_body
 
     # request_batch
     responses = [r async for r in scan.helpers.request_batch([url] * 100)]
     assert len(responses) == 100
     assert all([r[0] == url for r in responses])
-    assert all([r[1].status_code > 0 and r[1].text == "hello_there" for r in responses])
+    assert all([r[1].status_code > 0 and r[1].text == web_body for r in responses])
+
+    # request_batch w/ cancellation
+    agen = scan.helpers.request_batch([url] * 100)
+    async for url, response in agen:
+        assert response.text == web_body
+        await agen.aclose()
+        break
+
+    import asyncio
+
+    await asyncio.sleep(5)
 
     # download
     filename = await scan.helpers.download(url)
     file_content = open(filename).read()
-    assert file_content == "hello_there"
+    assert file_content == web_body
 
 
 @pytest.mark.asyncio

From 65f552a60163df880ce789c5e2505ce729a4f8f6 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 9 May 2024 09:52:23 -0400
Subject: [PATCH 090/220] fix ssl context

---
 bbot/core/helpers/web/engine.py      |  4 +++-
 bbot/core/helpers/web/ssl_context.py |  8 ++++++++
 bbot/modules/sslcert.py              | 12 ++++--------
 3 files changed, 15 insertions(+), 9 deletions(-)
 create mode 100644 bbot/core/helpers/web/ssl_context.py

diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index fd35a1f5d..bccf199d3 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -36,7 +36,9 @@ def __init__(self, socket_path, target, config={}):
         self._ssl_context_noverify = None
         self.ssl_verify = self.config.get("ssl_verify", False)
         if self.ssl_verify is False:
-            self.ssl_verify = self.ssl_context_noverify()
+            from .ssl_context import ssl_context_noverify
+
+            self.ssl_verify = ssl_context_noverify
         self.web_client = self.AsyncClient(persist_cookies=False)
 
     def AsyncClient(self, *args, **kwargs):
diff --git a/bbot/core/helpers/web/ssl_context.py b/bbot/core/helpers/web/ssl_context.py
new file mode 100644
index 000000000..fabe4188f
--- /dev/null
+++ b/bbot/core/helpers/web/ssl_context.py
@@ -0,0 +1,8 @@
+import ssl
+
+ssl_context_noverify = ssl.create_default_context()
+ssl_context_noverify.check_hostname = False
+ssl_context_noverify.verify_mode = ssl.CERT_NONE
+ssl_context_noverify.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3
+ssl_context_noverify.set_ciphers("ALL:@SECLEVEL=0")
+ssl_context_noverify.options |= 0x4  # Add the OP_LEGACY_SERVER_CONNECT option
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 42f34d23e..2440bbcd4 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -5,6 +5,7 @@
 from bbot.errors import ValidationError
 from bbot.modules.base import BaseModule
 from bbot.core.helpers.async_helpers import NamedLock
+from bbot.core.helpers.web.ssl_context import ssl_context_noverify
 
 
 class sslcert(BaseModule):
@@ -109,17 +110,12 @@ async def visit_host(self, host, port):
 
             host = str(host)
 
-            # Create an SSL context
-            try:
-                ssl_context = self.helpers.ssl_context_noverify()
-            except Exception as e:
-                self.warning(f"Error creating SSL context: {e}")
-                return [], [], (host, port)
-
             # Connect to the host
             try:
                 transport, _ = await asyncio.wait_for(
-                    self.helpers.loop.create_connection(lambda: asyncio.Protocol(), host, port, ssl=ssl_context),
+                    self.helpers.loop.create_connection(
+                        lambda: asyncio.Protocol(), host, port, ssl=ssl_context_noverify
+                    ),
                     timeout=self.timeout,
                 )
             except asyncio.TimeoutError:

From 3dad40a48be210463a8f30eb5cae2709787a7ab4 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 9 May 2024 17:29:00 -0400
Subject: [PATCH 091/220] fix scope bug

---
 bbot/scanner/preset/preset.py         | 23 +++++--
 bbot/test/test_step_1/test_cli.py     | 91 ++++++++++++++++++++++++---
 bbot/test/test_step_1/test_presets.py | 86 +++++++++++++++++++++++++
 3 files changed, 185 insertions(+), 15 deletions(-)

diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 9c8f12fbe..4f4138a06 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -240,9 +240,8 @@ def __init__(
         from bbot.scanner.target import Target
 
         self.target = Target(*targets, strict_scope=self.strict_scope)
-        if not whitelist:
-            self.whitelist = self.target.copy()
-        else:
+        self.whitelist = None
+        if whitelist:
             self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
         if not blacklist:
             blacklist = []
@@ -326,11 +325,16 @@ def merge(self, other):
         self.flags.update(other.flags)
         # scope
         self.target.add(other.target)
-        self.whitelist.add(other.whitelist)
+        if other.whitelist:
+            if self.whitelist is None:
+                self.whitelist = other.whitelist.copy()
+            else:
+                self.whitelist.add(other.whitelist)
         self.blacklist.add(other.blacklist)
         self.strict_scope = self.strict_scope or other.strict_scope
         for t in (self.target, self.whitelist):
-            t.strict_scope = self.strict_scope
+            if t is not None:
+                t.strict_scope = self.strict_scope
         # log verbosity
         if other.silent:
             self.silent = other.silent
@@ -377,6 +381,10 @@ def bake(self):
         os.environ.clear()
         os.environ.update(os_environ)
 
+        # ensure whitelist
+        if baked_preset.whitelist is None:
+            baked_preset.whitelist = baked_preset.target.copy()
+
         # validate flags, config options
         baked_preset.validate()
 
@@ -615,8 +623,11 @@ def whitelisted(self, host):
             >>> preset.whitelisted("http://www.evilcorp.com")
             True
         """
+        whitelist = self.whitelist
+        if whitelist is None:
+            whitelist = self.target
         e = make_event(host, dummy=True)
-        return e in self.whitelist
+        return e in whitelist
 
     @classmethod
     def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 0d36408c0..4b1f13143 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -1,10 +1,90 @@
 from ..bbot_fixtures import *
 
+from bbot import cli
+
 
 @pytest.mark.asyncio
-async def test_cli_scan(monkeypatch):
-    from bbot import cli
+async def test_cli_scope(monkeypatch, capsys):
+    import json
+
+    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
+    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
+
+    # basic target without whitelist
+    monkeypatch.setattr(
+        "sys.argv",
+        ["bbot", "-t", "one.one.one.one", "-c", "scope_report_distance=10", "dns_resolution=true", "--json"],
+    )
+    result = await cli._main()
+    out, err = capsys.readouterr()
+    assert result == True
+    lines = [json.loads(l) for l in out.splitlines()]
+    dns_events = [l for l in lines if l["type"] == "DNS_NAME" and l["data"] == "one.one.one.one"]
+    assert dns_events
+    assert all([l["scope_distance"] == 0 and "in-scope" in l["tags"] for l in dns_events])
+    assert 1 == len(
+        [
+            l
+            for l in dns_events
+            if l["module"] == "TARGET"
+            and l["scope_distance"] == 0
+            and "in-scope" in l["tags"]
+            and "target" in l["tags"]
+        ]
+    )
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.1.1.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in ip_events])
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.0.0.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in ip_events])
+
+    # with whitelist
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "bbot",
+            "-t",
+            "one.one.one.one",
+            "-w",
+            "192.168.0.1",
+            "-c",
+            "scope_report_distance=10",
+            "dns_resolution=true",
+            "scope_dns_search_distance=2",
+            "--json",
+        ],
+    )
+    result = await cli._main()
+    out, err = capsys.readouterr()
+    assert result == True
+    lines = [json.loads(l) for l in out.splitlines()]
+    lines = [l for l in lines if l["type"] != "SCAN"]
+    assert lines
+    assert not any([l["scope_distance"] == 0 for l in lines])
+    dns_events = [l for l in lines if l["type"] == "DNS_NAME" and l["data"] == "one.one.one.one"]
+    assert dns_events
+    assert all([l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in dns_events])
+    assert 1 == len(
+        [
+            l
+            for l in dns_events
+            if l["module"] == "TARGET"
+            and l["scope_distance"] == 1
+            and "distance-1" in l["tags"]
+            and "target" in l["tags"]
+        ]
+    )
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.1.1.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 2 and "distance-2" in l["tags"] for l in ip_events])
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.0.0.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 2 and "distance-2" in l["tags"] for l in ip_events])
 
+
+@pytest.mark.asyncio
+async def test_cli_scan(monkeypatch):
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
 
@@ -50,8 +130,6 @@ async def test_cli_scan(monkeypatch):
 
 @pytest.mark.asyncio
 async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
-    from bbot import cli
-
     caplog.set_level(logging.INFO)
 
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
@@ -315,8 +393,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
 
 
 def test_cli_config_validation(monkeypatch, caplog):
-    from bbot import cli
-
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
 
@@ -338,8 +414,6 @@ def test_cli_config_validation(monkeypatch, caplog):
 
 
 def test_cli_module_validation(monkeypatch, caplog):
-    from bbot import cli
-
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
 
@@ -458,7 +532,6 @@ def test_cli_module_validation(monkeypatch, caplog):
 
 def test_cli_presets(monkeypatch, capsys, caplog):
     import yaml
-    from bbot import cli
 
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py
index 479b48e85..4d159a82b 100644
--- a/bbot/test/test_step_1/test_presets.py
+++ b/bbot/test/test_step_1/test_presets.py
@@ -212,6 +212,92 @@ def test_preset_scope():
     preset1.merge(preset4)
     set(preset1.output_modules) == {"python", "csv", "txt", "json", "stdout", "neo4j"}
 
+    # test preset merging + whitelist
+
+    preset_nowhitelist = Preset("evilcorp.com")
+    preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
+    assert preset_nowhitelist.in_scope("www.evilcorp.com")
+    assert not preset_nowhitelist.in_scope("www.evilcorp.de")
+    assert not preset_nowhitelist.in_scope("1.2.3.4/24")
+
+    assert "www.evilcorp.org" in preset_whitelist.target
+    assert "1.2.3.4" in preset_whitelist.whitelist
+    assert not preset_whitelist.in_scope("www.evilcorp.org")
+    assert not preset_whitelist.in_scope("www.evilcorp.de")
+    assert not preset_whitelist.whitelisted("www.evilcorp.org")
+    assert not preset_whitelist.whitelisted("www.evilcorp.de")
+    assert preset_whitelist.in_scope("1.2.3.4")
+    assert preset_whitelist.in_scope("1.2.3.4/28")
+    assert preset_whitelist.in_scope("1.2.3.4/24")
+    assert preset_whitelist.whitelisted("1.2.3.4")
+    assert preset_whitelist.whitelisted("1.2.3.4/28")
+    assert preset_whitelist.whitelisted("1.2.3.4/24")
+
+    assert set([e.data for e in preset_nowhitelist.target]) == {"evilcorp.com"}
+    assert preset_nowhitelist.whitelist is None
+    assert set([e.data for e in preset_whitelist.target]) == {"evilcorp.org"}
+    baked_nowhitelist = preset_nowhitelist.bake()
+    assert set([e.data for e in baked_nowhitelist.whitelist]) == {"evilcorp.com"}
+    baked_whitelist = preset_whitelist.bake()
+    assert set([e.data for e in baked_whitelist.whitelist]) == {"1.2.3.0/24"}
+
+    preset_nowhitelist.merge(preset_whitelist)
+    assert set([e.data for e in preset_nowhitelist.target]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_nowhitelist.whitelist]) == {"1.2.3.0/24"}
+    assert "www.evilcorp.org" in preset_nowhitelist.target
+    assert "www.evilcorp.com" in preset_nowhitelist.target
+    assert "1.2.3.4" in preset_nowhitelist.whitelist
+    assert not preset_nowhitelist.in_scope("www.evilcorp.org")
+    assert not preset_nowhitelist.in_scope("www.evilcorp.com")
+    assert not preset_nowhitelist.whitelisted("www.evilcorp.org")
+    assert not preset_nowhitelist.whitelisted("www.evilcorp.com")
+    assert preset_nowhitelist.in_scope("1.2.3.4")
+
+    preset_nowhitelist = Preset("evilcorp.com")
+    preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
+    preset_whitelist.merge(preset_nowhitelist)
+    assert set([e.data for e in preset_whitelist.target]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_whitelist.whitelist]) == {"1.2.3.0/24"}
+    assert "www.evilcorp.org" in preset_whitelist.target
+    assert "www.evilcorp.com" in preset_whitelist.target
+    assert "1.2.3.4" in preset_whitelist.whitelist
+    assert not preset_whitelist.in_scope("www.evilcorp.org")
+    assert not preset_whitelist.in_scope("www.evilcorp.com")
+    assert not preset_whitelist.whitelisted("www.evilcorp.org")
+    assert not preset_whitelist.whitelisted("www.evilcorp.com")
+    assert preset_whitelist.in_scope("1.2.3.4")
+
+    preset_nowhitelist1 = Preset("evilcorp.com")
+    preset_nowhitelist2 = Preset("evilcorp.de")
+    assert set([e.data for e in preset_nowhitelist1.target]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2.target]) == {"evilcorp.de"}
+    assert preset_nowhitelist1.whitelist is None
+    assert preset_nowhitelist2.whitelist is None
+    preset_nowhitelist1.merge(preset_nowhitelist2)
+    assert set([e.data for e in preset_nowhitelist1.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist2.target]) == {"evilcorp.de"}
+    assert preset_nowhitelist1.whitelist is None
+    assert preset_nowhitelist2.whitelist is None
+    assert "www.evilcorp.com" in preset_nowhitelist1.target
+    assert "www.evilcorp.de" in preset_nowhitelist1.target
+    assert preset_nowhitelist1.whitelisted("www.evilcorp.com")
+    assert preset_nowhitelist1.whitelisted("www.evilcorp.de")
+    assert not preset_nowhitelist1.whitelisted("1.2.3.4")
+    assert preset_nowhitelist1.in_scope("www.evilcorp.com")
+    assert preset_nowhitelist1.in_scope("www.evilcorp.de")
+    assert not preset_nowhitelist1.in_scope("1.2.3.4")
+
+    preset_nowhitelist1 = Preset("evilcorp.com")
+    preset_nowhitelist2 = Preset("evilcorp.de")
+    preset_nowhitelist2.merge(preset_nowhitelist1)
+    assert set([e.data for e in preset_nowhitelist1.target]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert preset_nowhitelist1.whitelist is None
+    assert preset_nowhitelist2.whitelist is None
+    baked_nowhitelist2 = preset_nowhitelist2.bake()
+    assert set([e.data for e in baked_nowhitelist2.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in baked_nowhitelist2.whitelist]) == {"evilcorp.com", "evilcorp.de"}
+
 
 def test_preset_logging():
     # test verbosity levels (conflicting verbose/debug/silent)

From 5b731308e8bc498c7f8bd320e7d127a6b84a7177 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Fri, 10 May 2024 13:59:10 -0400
Subject: [PATCH 092/220] adding custom headers CLI argument / tests

---
 bbot/scanner/preset/args.py       | 24 ++++++++++++++++++++
 bbot/test/test_step_1/test_cli.py | 37 +++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py
index 11df99022..b39ce0896 100644
--- a/bbot/scanner/preset/args.py
+++ b/bbot/scanner/preset/args.py
@@ -145,6 +145,7 @@ def preset_from_args(self):
         args_preset.scan_name = self.parsed.name
         args_preset.output_dir = self.parsed.output_dir
         args_preset.force_start = self.parsed.force
+        args_preset.core.merge_custom({"http_headers": self.parsed.custom_headers})
 
         # CLI config options (dot-syntax)
         for config_arg in self.parsed.config:
@@ -299,6 +300,13 @@ def create_parser(self, *args, **kwargs):
 
         misc = p.add_argument_group(title="Misc")
         misc.add_argument("--version", action="store_true", help="show BBOT version and exit")
+        misc.add_argument(
+            "-H",
+            "--custom-headers",
+            nargs="+",
+            default=[],
+            help="List of custom headers as key value pairs (header=value).",
+        )
         return p
 
     def sanitize_args(self):
@@ -323,6 +331,22 @@ def sanitize_args(self):
         self.parsed.require_flags = chain_lists(self.parsed.require_flags)
         self.parsed.event_types = [t.upper() for t in chain_lists(self.parsed.event_types)]
 
+        # Custom Header Parsing / Validation
+        custom_headers_dict = {}
+        custom_header_example = "Example: --custom-headers foo=bar foo2=bar2"
+
+        for i in self.parsed.custom_headers:
+            parts = i.split("=", 1)
+            if len(parts) != 2:
+                raise ValidationError(f"Custom headers not formatted correctly (missing '='). {custom_header_example}")
+            k, v = parts
+            if not k or not v:
+                raise ValidationError(
+                    f"Custom headers not formatted correctly (missing header name or value). {custom_header_example}"
+                )
+            custom_headers_dict[k] = v
+        self.parsed.custom_headers = custom_headers_dict
+
     def validate(self):
         # validate config options
         sentinel = object()
diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 4b1f13143..1b1cfd035 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -392,6 +392,43 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert success == True, "--install-all-deps failed for at least one module"
 
 
+@pytest.mark.asyncio
+async def test_cli_customheaders(monkeypatch, caplog, capsys):
+    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
+    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
+    import yaml
+
+    # test custom headers
+    monkeypatch.setattr(
+        "sys.argv", ["bbot", "--custom-headers", "foo=bar", "foo2=bar2", "foo3=bar=3", "--current-preset"]
+    )
+    success = await cli._main()
+    assert success == None, "setting custom headers on command line failed"
+    captured = capsys.readouterr()
+    stdout_preset = yaml.safe_load(captured.out)
+    assert stdout_preset["config"]["http_headers"] == {"foo": "bar", "foo2": "bar2", "foo3": "bar=3"}
+
+    # test custom headers invalid (no "=")
+    monkeypatch.setattr("sys.argv", ["bbot", "--custom-headers", "justastring", "--current-preset"])
+    result = await cli._main()
+    assert result == None
+    assert "Custom headers not formatted correctly (missing '=')" in caplog.text
+    caplog.clear()
+
+    # test custom headers invalid (missing key)
+    monkeypatch.setattr("sys.argv", ["bbot", "--custom-headers", "=nokey", "--current-preset"])
+    result = await cli._main()
+    assert result == None
+    assert "Custom headers not formatted correctly (missing header name or value)" in caplog.text
+    caplog.clear()
+
+    # test custom headers invalid (missing value)
+    monkeypatch.setattr("sys.argv", ["bbot", "--custom-headers", "missingvalue=", "--current-preset"])
+    result = await cli._main()
+    assert result == None
+    assert "Custom headers not formatted correctly (missing header name or value)" in caplog.text
+
+
 def test_cli_config_validation(monkeypatch, caplog):
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)

From 7d8e0a5f84cb2162e31e25c6f1efb765c809806a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 10 May 2024 14:06:06 -0400
Subject: [PATCH 093/220] fix scope error

---
 bbot/scanner/preset/preset.py | 16 ++++++++--------
 bbot/scanner/target.py        | 19 +++++++------------
 2 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index fff1bbb76..29c92428d 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -324,15 +324,15 @@ def merge(self, other):
         self.explicit_output_modules.update(other.explicit_output_modules)
         self.flags.update(other.flags)
         # scope
-        self.target.target.add(other.target.target)
+        self.target.seeds.add(other.target.seeds)
         if other.whitelist:
-            if self.whitelist is None:
-                self.whitelist = other.whitelist.copy()
+            if self.target.whitelist is None:
+                self.target.whitelist = other.whitelist.copy()
             else:
                 self.whitelist.add(other.whitelist)
         self.blacklist.add(other.blacklist)
         self.strict_scope = self.strict_scope or other.strict_scope
-        for t in (self.target.target, self.whitelist):
+        for t in (self.target.seeds, self.target.whitelist):
             if t is not None:
                 t.strict_scope = self.strict_scope
         # log verbosity
@@ -382,8 +382,8 @@ def bake(self):
         os.environ.update(os_environ)
 
         # ensure whitelist
-        if baked_preset.whitelist is None:
-            baked_preset.whitelist = baked_preset.target.copy()
+        if baked_preset.target.whitelist is None:
+            baked_preset.target.whitelist = baked_preset.target.target.copy()
 
         # validate flags, config options
         baked_preset.validate()
@@ -581,7 +581,7 @@ def blacklisted(self, host):
     def whitelisted(self, host):
         whitelist = self.whitelist
         if whitelist is None:
-            whitelist = self.target.target
+            whitelist = self.target.seeds
         return host in whitelist
 
     @classmethod
@@ -699,7 +699,7 @@ def to_dict(self, include_target=False, full_config=False):
 
         # scope
         if include_target:
-            target = sorted(str(t.data) for t in self.target)
+            target = sorted(str(t.data) for t in self.target.seeds)
             whitelist = sorted(str(t.data) for t in self.target.whitelist)
             blacklist = sorted(str(t.data) for t in self.target.blacklist)
             if target:
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index ed0fc6066..ee23cb300 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -22,9 +22,9 @@ class BBOTTarget:
 
     def __init__(self, targets, whitelist=None, blacklist=None, strict_scope=False):
         self.strict_scope = strict_scope
-        self.target = Target(*targets, strict_scope=self.strict_scope)
+        self.seeds = Target(*targets, strict_scope=self.strict_scope)
         if whitelist is None:
-            self.whitelist = self.target.copy()
+            self.whitelist = None
         else:
             self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
         if blacklist is None:
@@ -32,17 +32,17 @@ def __init__(self, targets, whitelist=None, blacklist=None, strict_scope=False):
         self.blacklist = Target(*blacklist)
 
     def __iter__(self):
-        return iter(self.target)
+        return iter(self.seeds)
 
     def __len__(self):
-        return len(self.target)
+        return len(self.seeds)
 
     def __contains__(self, other):
-        return other in self.target
+        return other in self.seeds
 
     @property
     def events(self):
-        return self.target.events
+        return self.seeds.events
 
     def in_scope(self, host):
         """
@@ -104,16 +104,11 @@ def radix_only(self):
         A slimmer, serializable version of the target designed for simple scope checks
         """
         return self.__class__(
-            targets=[e.host for e in self.target if e.host],
+            targets=[e.host for e in self.seeds if e.host],
             whitelist=[e.host for e in self.whitelist if e.host],
             blacklist=[e.host for e in self.blacklist if e.host],
             strict_scope=self.strict_scope,
         )
-        # class RadixOnlyTarget:
-        #     def __init__(self, targets, whitelist, blacklist):
-        #         self.target = Target(*targets, strict_scope=self.strict_scope)
-        #         self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
-        #         self.blacklist = Target(*blacklist)
 
 
 class Target:

From 8016d53a5531e4d980c946bfbaec7114fd977c07 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 10 May 2024 14:15:24 -0400
Subject: [PATCH 094/220] fix tests again

---
 bbot/scanner/preset/preset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 29c92428d..e184112a8 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -383,7 +383,7 @@ def bake(self):
 
         # ensure whitelist
         if baked_preset.target.whitelist is None:
-            baked_preset.target.whitelist = baked_preset.target.target.copy()
+            baked_preset.target.whitelist = baked_preset.target.seeds.copy()
 
         # validate flags, config options
         baked_preset.validate()

From 05451afcb6ee1a4207030c87ab80ce4fd53a4c48 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 10 May 2024 17:10:23 -0400
Subject: [PATCH 095/220] more test fixes, telerik

---
 bbot/core/engine.py                   |  40 +++++-----
 bbot/core/helpers/web/engine.py       |   3 +-
 bbot/modules/telerik.py               |  33 ++++----
 bbot/scanner/preset/preset.py         |  15 ++--
 bbot/scanner/target.py                |  13 ++-
 bbot/test/test_step_1/test_engine.py  | 111 +++++++++++++++++++++++---
 bbot/test/test_step_1/test_presets.py |   8 +-
 7 files changed, 163 insertions(+), 60 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 2e08ca894..412a5b26d 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -46,6 +46,8 @@ def check_error(self, message):
         if message is error_sentinel:
             return True
         if isinstance(message, dict) and len(message) == 1 and "_e" in message:
+            error = message["_e"][0]
+            raise BBOTEngineError(error)
             return True
         return False
 
@@ -201,11 +203,11 @@ def __init__(self, socket_path):
             self.tasks = dict()
 
     async def run_and_return(self, client_id, command_fn, *args, **kwargs):
-        self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({kwargs})")
+        self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({args}, {kwargs})")
         try:
             result = await command_fn(*args, **kwargs)
-        except ValueError as e:
-            error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
+        except BaseException as e:
+            error = f"Unhandled error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
             trace = traceback.format_exc()
             self.log.error(error)
             self.log.trace(trace)
@@ -215,13 +217,13 @@ async def run_and_return(self, client_id, command_fn, *args, **kwargs):
         await self.send_socket_multipart([client_id, pickle.dumps(result)])
 
     async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
-        self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({kwargs})")
+        self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({args}, {kwargs})")
         try:
             async for _ in command_fn(*args, **kwargs):
                 await self.send_socket_multipart([client_id, pickle.dumps(_)])
             await self.send_socket_multipart([client_id, pickle.dumps({"_s": None})])
-        except ValueError as e:
-            error = f"Unhandled error in {self.name}.{command_fn.__name__}({kwargs}): {e}"
+        except BaseException as e:
+            error = f"Unhandled error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
             trace = traceback.format_exc()
             self.log.error(error)
             self.log.trace(trace)
@@ -252,18 +254,20 @@ async def worker(self):
                     continue
 
                 if cmd == -1:
-                    task, _cmd, _args, _kwargs = self.tasks.get(client_id, None)
-                    if task is not None:
-                        self.log.debug(f"Cancelling client id {client_id} (task: {task})")
-                        task.cancel()
-                        try:
-                            await task
-                        except (KeyboardInterrupt, asyncio.CancelledError):
-                            pass
-                        except BaseException as e:
-                            self.log.error(f"Unhandled error in {_cmd}({_args}, {_kwargs}): {e}")
-                            self.log.trace(traceback.format_exc())
-                        self.tasks.pop(client_id, None)
+                    task = self.tasks.get(client_id, None)
+                    if task is None:
+                        continue
+                    task, _cmd, _args, _kwargs = task
+                    self.log.debug(f"Cancelling client id {client_id} (task: {task})")
+                    task.cancel()
+                    try:
+                        await task
+                    except (KeyboardInterrupt, asyncio.CancelledError):
+                        pass
+                    except BaseException as e:
+                        self.log.error(f"Unhandled error in {_cmd}({_args}, {_kwargs}): {e}")
+                        self.log.trace(traceback.format_exc())
+                    self.tasks.pop(client_id, None)
                     continue
 
                 args = message.get("a", ())
diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index bccf199d3..8c6cd59c9 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -110,8 +110,7 @@ def new_task(url):
             for task in done:
                 response = task.result()
                 url = tasks.pop(task)
-                if response:
-                    yield (url, response)
+                yield (url, response)
                 if urls:  # Start a new task for each one completed, if URLs remain
                     new_task(urls.pop(0))
 
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index c86da8897..0c2e3e839 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -1,4 +1,3 @@
-import asyncio
 from sys import executable
 from urllib.parse import urlparse
 
@@ -249,31 +248,25 @@ async def handle_event(self, event):
                                     )
                                     break
 
-            tasks = []
+            urls = {}
             for dh in self.DialogHandlerUrls:
-                tasks.append(self.helpers.create_task(self.test_detector(event.data, f"{dh}?dp=1")))
+                url = self.create_url(event.data, f"{dh}?dp=1")
+                urls[url] = dh
 
+            gen = self.helpers.request_batch(list(urls))
             fail_count = 0
-            gen = self.helpers.as_completed(tasks)
-            async for task in gen:
-                try:
-                    result, dh = await task
-                except asyncio.CancelledError:
-                    continue
-
+            async for url, response in gen:
                 # cancel if we run into timeouts etc.
-                if result is None:
+                if response is None:
                     fail_count += 1
 
                     # tolerate some random errors
                     if fail_count < 2:
                         continue
                     self.debug(f"Cancelling run against {event.data} due to failed request")
-                    await self.helpers.cancel_tasks(tasks)
                     await gen.aclose()
                 else:
-                    if "Cannot deserialize dialog parameters" in result.text:
-                        await self.helpers.cancel_tasks(tasks)
+                    if "Cannot deserialize dialog parameters" in response.text:
                         self.debug(f"Detected Telerik UI instance ({dh})")
                         description = f"Telerik DialogHandler detected"
                         await self.emit_event(
@@ -284,8 +277,6 @@ async def handle_event(self, event):
                         # Once we have a match we need to stop, because the basic handler (Telerik.Web.UI.DialogHandler.aspx) usually works with a path wildcard
                         await gen.aclose()
 
-            await self.helpers.cancel_tasks(tasks)
-
             spellcheckhandler = "Telerik.Web.UI.SpellCheckHandler.axd"
             result, _ = await self.test_detector(event.data, spellcheckhandler)
             status_code = getattr(result, "status_code", 0)
@@ -352,12 +343,16 @@ async def handle_event(self, event):
 
         # Check for RAD Controls in URL
 
-    async def test_detector(self, baseurl, detector):
-        result = None
-        if "/" != baseurl[-1]:
+    def create_url(self, baseurl, detector):
+        if not baseurl.endswith("/"):
             url = f"{baseurl}/{detector}"
         else:
             url = f"{baseurl}{detector}"
+        return url
+
+    async def test_detector(self, baseurl, detector):
+        result = None
+        url = self.create_url(baseurl, detector)
         result = await self.helpers.request(url, timeout=self.timeout)
         return result, detector
 
diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index e184112a8..901108536 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -325,8 +325,8 @@ def merge(self, other):
         self.flags.update(other.flags)
         # scope
         self.target.seeds.add(other.target.seeds)
-        if other.whitelist:
-            if self.target.whitelist is None:
+        if other.target.whitelist is not None:
+            if self.whitelist is None:
                 self.target.whitelist = other.whitelist.copy()
             else:
                 self.whitelist.add(other.whitelist)
@@ -369,6 +369,8 @@ def bake(self):
         baked_preset = copy(self)
         # copy core
         baked_preset.core = self.core.copy()
+        # copy target
+        baked_preset.target = self.target.copy()
         # copy module loader
         baked_preset._module_loader = self.module_loader.copy()
         # prepare os environment
@@ -579,10 +581,7 @@ def blacklisted(self, host):
         return self.target.blacklisted(host)
 
     def whitelisted(self, host):
-        whitelist = self.whitelist
-        if whitelist is None:
-            whitelist = self.target.seeds
-        return host in whitelist
+        return self.target.whitelisted(host)
 
     @classmethod
     def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
@@ -700,7 +699,9 @@ def to_dict(self, include_target=False, full_config=False):
         # scope
         if include_target:
             target = sorted(str(t.data) for t in self.target.seeds)
-            whitelist = sorted(str(t.data) for t in self.target.whitelist)
+            whitelist = set()
+            if self.target.whitelist is not None:
+                whitelist = sorted(str(t.data) for t in self.target.whitelist)
             blacklist = sorted(str(t.data) for t in self.target.blacklist)
             if target:
                 preset_dict["target"] = target
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index ee23cb300..ea423167d 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -40,6 +40,14 @@ def __len__(self):
     def __contains__(self, other):
         return other in self.seeds
 
+    def copy(self):
+        self_copy = copy.copy(self)
+        self_copy.seeds = self.seeds.copy()
+        if self.whitelist is not None:
+            self_copy.whitelist = self.whitelist.copy()
+        self_copy.blacklist = self.blacklist.copy()
+        return self_copy
+
     @property
     def events(self):
         return self.seeds.events
@@ -96,7 +104,10 @@ def whitelisted(self, host):
             True
         """
         e = make_event(host, dummy=True)
-        return e in self.whitelist
+        whitelist = self.whitelist
+        if whitelist is None:
+            whitelist = self.seeds
+        return e in whitelist
 
     @property
     def radix_only(self):
diff --git a/bbot/test/test_step_1/test_engine.py b/bbot/test/test_step_1/test_engine.py
index 5413e3b03..ff0f85174 100644
--- a/bbot/test/test_step_1/test_engine.py
+++ b/bbot/test/test_step_1/test_engine.py
@@ -3,9 +3,16 @@
 
 @pytest.mark.asyncio
 async def test_engine():
+    from bbot.errors import BBOTEngineError
     from bbot.core.engine import EngineClient, EngineServer
 
     counter = 0
+    yield_cancelled = False
+    yield_errored = False
+    return_started = False
+    return_finished = False
+    return_cancelled = False
+    return_errored = False
 
     class TestEngineServer(EngineServer):
 
@@ -14,34 +21,59 @@ class TestEngineServer(EngineServer):
             1: "yield_stuff",
         }
 
-        async def return_thing(self):
-            return "thing"
+        async def return_thing(self, n):
+            nonlocal return_started
+            nonlocal return_finished
+            nonlocal return_cancelled
+            nonlocal return_errored
+            try:
+                return_started = True
+                await asyncio.sleep(n)
+                return_finished = True
+                return f"thing{n}"
+            except asyncio.CancelledError:
+                return_cancelled = True
+                raise
+            except Exception:
+                return_errored = True
+                raise
 
         async def yield_stuff(self, n):
             nonlocal counter
-            for i in range(n):
-                yield f"thing{i}"
-                counter += 1
-                await asyncio.sleep(0.1)
+            nonlocal yield_cancelled
+            nonlocal yield_errored
+            try:
+                for i in range(n):
+                    yield f"thing{i}"
+                    counter += 1
+                    await asyncio.sleep(0.1)
+            except asyncio.CancelledError:
+                yield_cancelled = True
+                raise
+            except Exception:
+                yield_errored = True
+                raise
 
     class TestEngineClient(EngineClient):
 
         SERVER_CLASS = TestEngineServer
 
-        async def return_thing(self):
-            return await self.run_and_return("return_thing")
+        async def return_thing(self, n):
+            return await self.run_and_return("return_thing", n)
 
         async def yield_stuff(self, n):
             async for _ in self.run_and_yield("yield_stuff", n):
                 yield _
 
-    # test return functionality
     test_engine = TestEngineClient()
-    return_res = await test_engine.return_thing()
-    assert return_res == "thing"
+
+    # test return functionality
+    return_res = await test_engine.return_thing(1)
+    assert return_res == "thing1"
 
     # test async generator
     assert counter == 0
+    assert yield_cancelled == False
     yield_res = [r async for r in test_engine.yield_stuff(13)]
     assert yield_res == [f"thing{i}" for i in range(13)]
     assert len(yield_res) == 13
@@ -49,10 +81,67 @@ async def yield_stuff(self, n):
 
     # test async generator with cancellation
     counter = 0
+    yield_cancelled = False
+    yield_errored = False
     agen = test_engine.yield_stuff(1000)
     async for r in agen:
         if counter > 10:
             await agen.aclose()
             break
     await asyncio.sleep(5)
+    assert yield_cancelled == True
+    assert yield_errored == False
     assert counter < 15
+
+    # test async generator with error
+    yield_cancelled = False
+    yield_errored = False
+    agen = test_engine.yield_stuff(None)
+    with pytest.raises(BBOTEngineError):
+        async for _ in agen:
+            pass
+    assert yield_cancelled == False
+    assert yield_errored == True
+
+    # test return with cancellation
+    return_started = False
+    return_finished = False
+    return_cancelled = False
+    return_errored = False
+    task = asyncio.create_task(test_engine.return_thing(2))
+    await asyncio.sleep(1)
+    task.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await task
+    await asyncio.sleep(0.1)
+    assert return_started == True
+    assert return_finished == False
+    assert return_cancelled == True
+    assert return_errored == False
+
+    # test return with late cancellation
+    return_started = False
+    return_finished = False
+    return_cancelled = False
+    return_errored = False
+    task = asyncio.create_task(test_engine.return_thing(1))
+    await asyncio.sleep(2)
+    task.cancel()
+    result = await task
+    assert result == "thing1"
+    assert return_started == True
+    assert return_finished == True
+    assert return_cancelled == False
+    assert return_errored == False
+
+    # test return with error
+    return_started = False
+    return_finished = False
+    return_cancelled = False
+    return_errored = False
+    with pytest.raises(BBOTEngineError):
+        result = await test_engine.return_thing(None)
+    assert return_started == True
+    assert return_finished == False
+    assert return_cancelled == False
+    assert return_errored == True
diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py
index 4d159a82b..afa0e3ded 100644
--- a/bbot/test/test_step_1/test_presets.py
+++ b/bbot/test/test_step_1/test_presets.py
@@ -214,8 +214,12 @@ def test_preset_scope():
 
     # test preset merging + whitelist
 
-    preset_nowhitelist = Preset("evilcorp.com")
-    preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
+    preset_nowhitelist = Preset("evilcorp.com", name="nowhitelist")
+    preset_whitelist = Preset("evilcorp.org", name="whitelist", whitelist=["1.2.3.4/24"])
+
+    assert preset_nowhitelist.to_dict(include_target=True) == {"target": ["evilcorp.com"]}
+    assert preset_whitelist.to_dict(include_target=True) == {"target": ["evilcorp.org"], "whitelist": ["1.2.3.0/24"]}
+
     assert preset_nowhitelist.in_scope("www.evilcorp.com")
     assert not preset_nowhitelist.in_scope("www.evilcorp.de")
     assert not preset_nowhitelist.in_scope("1.2.3.4/24")

From 527ec4ead0f1a9faf3a0531b84f0fbccaacab318 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 00:28:07 -0400
Subject: [PATCH 096/220] more tests, optimizations

---
 bbot/cli.py                          |  2 --
 bbot/core/helpers/helper.py          | 11 ++++++++--
 bbot/modules/internal/cloud.py       | 15 +++++++++----
 bbot/scanner/preset/args.py          |  9 ++++----
 bbot/scanner/preset/preset.py        | 14 ++----------
 bbot/scanner/target.py               | 32 ++++++++++++++++++++++++++-
 bbot/test/test_step_1/test_target.py | 33 ++++++++++++++++++++++++++++
 7 files changed, 91 insertions(+), 25 deletions(-)

diff --git a/bbot/cli.py b/bbot/cli.py
index 5cf2f9cd1..7910d2a36 100755
--- a/bbot/cli.py
+++ b/bbot/cli.py
@@ -44,8 +44,6 @@ async def _main():
 
         # start by creating a default scan preset
         preset = Preset(_log=True, name="bbot_cli_main")
-        # populate preset symlinks
-        preset.all_presets
         # parse command line arguments and merge into preset
         try:
             preset.parse_args()
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index 56b9c3bbd..4ba21cdc5 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -4,7 +4,6 @@
 from pathlib import Path
 import multiprocessing as mp
 from functools import partial
-from cloudcheck import cloud_providers
 from concurrent.futures import ProcessPoolExecutor
 
 from . import misc
@@ -82,7 +81,7 @@ def __init__(self, preset):
         num_processes = max(1, mp.cpu_count() - 1)
         self.process_pool = ProcessPoolExecutor(max_workers=num_processes)
 
-        self.cloud = cloud_providers
+        self._cloud = None
 
         self.re = RegexHelper(self)
         self.dns = DNSHelper(self)
@@ -91,6 +90,14 @@ def __init__(self, preset):
         self.word_cloud = WordCloud(self)
         self.dummy_modules = {}
 
+    @property
+    def cloud(self):
+        if self._cloud is None:
+            from cloudcheck import cloud_providers
+
+            self._cloud = cloud_providers
+        return self._cloud
+
     def bloom_filter(self, size):
         from .bloom import BloomFilter
 
diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py
index 7939487fd..ac4eb71f7 100644
--- a/bbot/modules/internal/cloud.py
+++ b/bbot/modules/internal/cloud.py
@@ -8,12 +8,19 @@ class cloud(InterceptModule):
     _priority = 3
 
     async def setup(self):
-        self.dummy_modules = {}
-        for provider_name, provider in self.helpers.cloud.providers.items():
-            self.dummy_modules[provider_name] = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan")
-
+        self._dummy_modules = None
         return True
 
+    @property
+    def dummy_modules(self):
+        if self._dummy_modules is None:
+            self._dummy_modules = {}
+            for provider_name, provider in self.helpers.cloud.providers.items():
+                self._dummy_modules[provider_name] = self.scan._make_dummy_module(
+                    f"cloud_{provider_name}", _type="scan"
+                )
+        return self._dummy_modules
+
     async def filter_event(self, event):
         if (not event.host) or (event.type in ("IP_RANGE",)):
             return False, "event does not have host attribute"
diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py
index b39ce0896..24c5707e3 100644
--- a/bbot/scanner/preset/args.py
+++ b/bbot/scanner/preset/args.py
@@ -173,7 +173,7 @@ def create_parser(self, *args, **kwargs):
             "-w",
             "--whitelist",
             nargs="+",
-            default=[],
+            default=None,
             help="What's considered in-scope (by default it's the same as --targets)",
         )
         target.add_argument("-b", "--blacklist", nargs="+", default=[], help="Don't touch these things")
@@ -320,9 +320,10 @@ def sanitize_args(self):
         self.parsed.targets = chain_lists(
             self.parsed.targets, try_files=True, msg="Reading targets from file: {filename}"
         )
-        self.parsed.whitelist = chain_lists(
-            self.parsed.whitelist, try_files=True, msg="Reading whitelist from file: {filename}"
-        )
+        if self.parsed.whitelist is not None:
+            self.parsed.whitelist = chain_lists(
+                self.parsed.whitelist, try_files=True, msg="Reading whitelist from file: {filename}"
+            )
         self.parsed.blacklist = chain_lists(
             self.parsed.blacklist, try_files=True, msg="Reading blacklist from file: {filename}"
         )
diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 901108536..ac42ba99b 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -323,18 +323,8 @@ def merge(self, other):
         self.explicit_scan_modules.update(other.explicit_scan_modules)
         self.explicit_output_modules.update(other.explicit_output_modules)
         self.flags.update(other.flags)
-        # scope
-        self.target.seeds.add(other.target.seeds)
-        if other.target.whitelist is not None:
-            if self.whitelist is None:
-                self.target.whitelist = other.whitelist.copy()
-            else:
-                self.whitelist.add(other.whitelist)
-        self.blacklist.add(other.blacklist)
-        self.strict_scope = self.strict_scope or other.strict_scope
-        for t in (self.target.seeds, self.target.whitelist):
-            if t is not None:
-                t.strict_scope = self.strict_scope
+        # target / scope
+        self.target.merge(other.target)
         # log verbosity
         if other.silent:
             self.silent = other.silent
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index ea423167d..531ae9774 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -31,6 +31,25 @@ def __init__(self, targets, whitelist=None, blacklist=None, strict_scope=False):
             blacklist = []
         self.blacklist = Target(*blacklist)
 
+    def merge(self, other):
+        self.seeds.add(other.seeds)
+        if other.whitelist is not None:
+            if self.whitelist is None:
+                self.whitelist = other.whitelist.copy()
+            else:
+                self.whitelist.add(other.whitelist)
+        self.blacklist.add(other.blacklist)
+        self.strict_scope = self.strict_scope or other.strict_scope
+        for t in (self.seeds, self.whitelist):
+            if t is not None:
+                t.strict_scope = self.strict_scope
+
+    def get(self, host):
+        return self.seeds.get(host)
+
+    def get_host(self, host):
+        return self.seeds.get(host)
+
     def __iter__(self):
         return iter(self.seeds)
 
@@ -38,8 +57,19 @@ def __len__(self):
         return len(self.seeds)
 
     def __contains__(self, other):
+        if isinstance(other, self.__class__):
+            other = other.seeds
         return other in self.seeds
 
+    def __bool__(self):
+        return bool(self.seeds)
+
+    def __eq__(self, other):
+        return hash(self) == hash(other)
+
+    def __hash__(self):
+        return hash(self.seeds)
+
     def copy(self):
         self_copy = copy.copy(self)
         self_copy.seeds = self.seeds.copy()
@@ -367,7 +397,7 @@ def __iter__(self):
 
     def __contains__(self, other):
         # if "other" is a Target
-        if type(other) == self.__class__:
+        if isinstance(other, self.__class__):
             contained_in_self = [self._contains(e) for e in other.events]
             return all(contained_in_self)
         else:
diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py
index 7d8117d52..c2f13a497 100644
--- a/bbot/test/test_step_1/test_target.py
+++ b/bbot/test/test_step_1/test_target.py
@@ -32,6 +32,39 @@ def test_target(bbot_scanner):
     assert scan2.target == scan3.target
     assert scan4.target != scan1.target
 
+    assert not scan5.target.seeds
+    assert len(scan1.target.seeds) == 9
+    assert len(scan4.target.seeds) == 8
+    assert "8.8.8.9" in scan1.target.seeds
+    assert "8.8.8.12" not in scan1.target.seeds
+    assert "8.8.8.8/31" in scan1.target.seeds
+    assert "8.8.8.8/30" in scan1.target.seeds
+    assert "8.8.8.8/29" not in scan1.target.seeds
+    assert "2001:4860:4860::8889" in scan1.target.seeds
+    assert "2001:4860:4860::888c" not in scan1.target.seeds
+    assert "www.api.publicapis.org" in scan1.target.seeds
+    assert "api.publicapis.org" in scan1.target.seeds
+    assert "publicapis.org" not in scan1.target.seeds
+    assert "bob@www.api.publicapis.org" in scan1.target.seeds
+    assert "https://www.api.publicapis.org" in scan1.target.seeds
+    assert "www.api.publicapis.org:80" in scan1.target.seeds
+    assert scan1.make_event("https://[2001:4860:4860::8888]:80", dummy=True) in scan1.target.seeds
+    assert scan1.make_event("[2001:4860:4860::8888]:80", "OPEN_TCP_PORT", dummy=True) in scan1.target.seeds
+    assert scan1.make_event("[2001:4860:4860::888c]:80", "OPEN_TCP_PORT", dummy=True) not in scan1.target.seeds
+
+    assert scan1.whitelisted("https://[2001:4860:4860::8888]:80")
+    assert scan1.whitelisted("[2001:4860:4860::8888]:80")
+    assert not scan1.whitelisted("[2001:4860:4860::888c]:80")
+    assert scan1.whitelisted("www.api.publicapis.org")
+    assert scan1.whitelisted("api.publicapis.org")
+    assert not scan1.whitelisted("publicapis.org")
+
+    assert scan1.target.seeds in scan2.target.seeds
+    assert scan2.target.seeds not in scan1.target.seeds
+    assert scan3.target.seeds in scan2.target.seeds
+    assert scan2.target.seeds == scan3.target.seeds
+    assert scan4.target.seeds != scan1.target.seeds
+
     assert str(scan1.target.get("8.8.8.9").host) == "8.8.8.8/30"
     assert scan1.target.get("8.8.8.12") is None
     assert str(scan1.target.get("2001:4860:4860::8889").host) == "2001:4860:4860::8888/126"

From eb0c2af0e467406fbcedd34efd17da9a635a901b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 00:44:22 -0400
Subject: [PATCH 097/220] test debugging

---
 bbot/test/test_step_1/test_manager_deduplication.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 3f88576f9..f607de359 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -117,6 +117,8 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
+    for e in all_events:
+        log.critical(e)
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])

From 79f7c000af888339880627d099534a0859d42e9c Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 01:25:17 -0400
Subject: [PATCH 098/220] fix tests

---
 .../test_step_1/test_manager_deduplication.py | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index f607de359..95c3bb55c 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -75,13 +75,13 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
         )
 
     dns_mock_chain = {
-        "test.notreal": {"A": ["127.0.0.3"]},
-        "default_module.test.notreal": {"A": ["127.0.0.3"]},
-        "everything_module.test.notreal": {"A": ["127.0.0.4"]},
-        "no_suppress_dupes.test.notreal": {"A": ["127.0.0.5"]},
-        "accept_dupes.test.notreal": {"A": ["127.0.0.6"]},
-        "per_hostport_only.test.notreal": {"A": ["127.0.0.7"]},
-        "per_domain_only.test.notreal": {"A": ["127.0.0.8"]},
+        "test.notreal": {"A": ["169.169.0.3"]},
+        "default_module.test.notreal": {"A": ["169.169.0.3"]},
+        "everything_module.test.notreal": {"A": ["169.169.0.4"]},
+        "no_suppress_dupes.test.notreal": {"A": ["169.169.0.5"]},
+        "accept_dupes.test.notreal": {"A": ["169.169.0.6"]},
+        "per_hostport_only.test.notreal": {"A": ["169.169.0.7"]},
+        "per_domain_only.test.notreal": {"A": ["169.169.0.8"]},
     }
 
     # dns search distance = 1, report distance = 0
@@ -118,7 +118,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
     for e in all_events:
-        log.critical(e)
+        log.critical(f"{e} <-- {e.source}")
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
@@ -130,12 +130,12 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.8" and str(e.module) == "A" and e.source.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.3" and str(e.module) == "A" and e.source.data == "test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.7" and str(e.module) == "A" and e.source.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.8" and str(e.module) == "A" and e.source.data == "per_domain_only.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_domain_only.test.notreal"])

From f64c8101d0b25453badec1d5585a627671015d6c Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 01:35:58 -0400
Subject: [PATCH 099/220] troubleshooting

---
 .../test_step_1/test_manager_deduplication.py | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 95c3bb55c..2a06cb530 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -75,13 +75,13 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
         )
 
     dns_mock_chain = {
-        "test.notreal": {"A": ["169.169.0.3"]},
-        "default_module.test.notreal": {"A": ["169.169.0.3"]},
-        "everything_module.test.notreal": {"A": ["169.169.0.4"]},
-        "no_suppress_dupes.test.notreal": {"A": ["169.169.0.5"]},
-        "accept_dupes.test.notreal": {"A": ["169.169.0.6"]},
-        "per_hostport_only.test.notreal": {"A": ["169.169.0.7"]},
-        "per_domain_only.test.notreal": {"A": ["169.169.0.8"]},
+        "test.notreal": {"A": ["127.0.0.3"]},
+        "default_module.test.notreal": {"A": ["127.0.0.3"]},
+        "everything_module.test.notreal": {"A": ["127.0.0.4"]},
+        "no_suppress_dupes.test.notreal": {"A": ["127.0.0.5"]},
+        "accept_dupes.test.notreal": {"A": ["127.0.0.6"]},
+        "per_hostport_only.test.notreal": {"A": ["127.0.0.7"]},
+        "per_domain_only.test.notreal": {"A": ["127.0.0.8"]},
     }
 
     # dns search distance = 1, report distance = 0
@@ -130,12 +130,12 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.3" and str(e.module) == "A" and e.source.data == "test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.7" and str(e.module) == "A" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "169.169.0.8" and str(e.module) == "A" and e.source.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.source.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.8" and str(e.module) == "A" and e.source.data == "per_domain_only.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_domain_only.test.notreal"])

From f913f68988ec90c3bea7509a8c62b14e4cfbf698 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 01:57:42 -0400
Subject: [PATCH 100/220] huh wat

---
 bbot/test/test_step_1/test_manager_deduplication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 2a06cb530..508c1ed74 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -118,7 +118,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
     for e in all_events:
-        log.critical(f"{e} <-- {e.source}")
+        log.critical(f"{e}:{e.always_emit} <-- {e.source}:{e.source.always_emit} <-- {e.source.source}:{e.source.source.always_emit}")
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])

From a801eee539b63627135d4f480f492646e7b84cba Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 02:15:53 -0400
Subject: [PATCH 101/220] fix tests

---
 bbot/test/test_step_1/test_cli.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 1b1cfd035..fbe4324bc 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -655,3 +655,6 @@ def test_cli_presets(monkeypatch, capsys, caplog):
     monkeypatch.setattr("sys.argv", ["bbot", "-p", "asdfasdfasdf", "-y"])
     cli.main()
     assert "file does not exist. Use -lp to list available presets" in caplog.text
+
+    preset1_file.unlink()
+    preset2_file.unlink()

From 78ad88c28299125e4d05be417a7808cf1416b44c Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 02:28:24 -0400
Subject: [PATCH 102/220] hmmmm

---
 bbot/scanner/manager.py                             | 7 +++++--
 bbot/test/test_step_1/test_manager_deduplication.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index c3210d65d..e18120560 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -69,8 +69,11 @@ async def handle_event(self, event, kwargs):
             return False, "event's source is itself"
 
         # don't accept duplicates
-        if (not event._graph_important) and self.is_incoming_duplicate(event, add=True):
-            return False, "event was already emitted by its module"
+        if self.is_incoming_duplicate(event, add=True):
+            if not event._graph_important:
+                return False, "event was already emitted by its module"
+            else:
+                self.debug(f"Event {event} was already emitted by its module, but this time it's graph-important")
 
         # update event's scope distance based on its parent
         event.scope_distance = event.source.scope_distance + 1
diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 508c1ed74..11fb524a0 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -118,7 +118,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
     for e in all_events:
-        log.critical(f"{e}:{e.always_emit} <-- {e.source}:{e.source.always_emit} <-- {e.source.source}:{e.source.source.always_emit}")
+        log.critical(f"{e}:{e.graph_important} <-- {e.source}:{e.source.graph_important} <-- {e.source.source}:{e.source.source.graph_important}")
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])

From 490c486a940e0af96bc1a49267884c4edbe0ddd3 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 02:38:21 -0400
Subject: [PATCH 103/220] hmmmm

---
 bbot/test/test_step_1/test_manager_deduplication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 11fb524a0..3dea17f3b 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -118,7 +118,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
     for e in all_events:
-        log.critical(f"{e}:{e.graph_important} <-- {e.source}:{e.source.graph_important} <-- {e.source.source}:{e.source.source.graph_important}")
+        log.critical(f"{e}:{e._graph_important} <-- {e.source}:{e.source._graph_important} <-- {e.source.source}:{e.source.source._graph_important}")
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])

From 6d6d6025e17f1dfc775f81c642cb52a757f86441 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 03:29:11 -0400
Subject: [PATCH 104/220] fix baddns tests

---
 bbot/core/helpers/web/client.py                   | 14 ++++++++++++++
 bbot/core/helpers/web/engine.py                   | 15 +--------------
 bbot/core/helpers/web/web.py                      |  9 ++++++++-
 .../module_tests/test_module_baddns.py            | 12 ++++++------
 4 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py
index 32cf47067..76015411c 100644
--- a/bbot/core/helpers/web/client.py
+++ b/bbot/core/helpers/web/client.py
@@ -28,6 +28,20 @@ class BBOTAsyncClient(httpx.AsyncClient):
         200
     """
 
+    @classmethod
+    def from_config(cls, config, target, *args, **kwargs):
+        kwargs["_config"] = config
+        kwargs["_target"] = target
+        retries = kwargs.pop("retries", config.get("http_retries", 1))
+        ssl_verify = config.get("ssl_verify", False)
+        if ssl_verify is False:
+            from .ssl_context import ssl_context_noverify
+
+            ssl_verify = ssl_context_noverify
+        kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=ssl_verify)
+        kwargs["verify"] = ssl_verify
+        return cls(*args, **kwargs)
+
     def __init__(self, *args, **kwargs):
         self._config = kwargs.pop("_config")
         self._target = kwargs.pop("_target")
diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 8c6cd59c9..09f8d9591 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -34,20 +34,7 @@ def __init__(self, socket_path, target, config={}):
         self.config = config
         self.http_debug = self.config.get("http_debug", False)
         self._ssl_context_noverify = None
-        self.ssl_verify = self.config.get("ssl_verify", False)
-        if self.ssl_verify is False:
-            from .ssl_context import ssl_context_noverify
-
-            self.ssl_verify = ssl_context_noverify
-        self.web_client = self.AsyncClient(persist_cookies=False)
-
-    def AsyncClient(self, *args, **kwargs):
-        kwargs["_config"] = self.config
-        kwargs["_target"] = self.target
-        retries = kwargs.pop("retries", self.config.get("http_retries", 1))
-        kwargs["transport"] = httpx.AsyncHTTPTransport(retries=retries, verify=self.ssl_verify)
-        kwargs["verify"] = self.ssl_verify
-        return BBOTAsyncClient(*args, **kwargs)
+        self.web_client = BBOTAsyncClient.from_config(self.config, self.target, persist_cookies=False)
 
     async def request(self, *args, **kwargs):
         raise_error = kwargs.pop("raise_error", False)
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 179abdc35..5cf7141fb 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -48,10 +48,17 @@ class WebHelper(EngineClient):
 
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
-        self.config = self.parent_helper.config
+        self.preset = self.parent_helper.preset
+        self.config = self.preset.config
+        self.target = self.preset.target
         self.ssl_verify = self.config.get("ssl_verify", False)
         super().__init__(server_kwargs={"config": self.config, "target": self.parent_helper.preset.target.radix_only})
 
+    def AsyncClient(self, *args, **kwargs):
+        from .client import BBOTAsyncClient
+
+        return BBOTAsyncClient.from_config(self.config, self.target, *args, persist_cookies=False, **kwargs)
+
     async def request(self, *args, **kwargs):
         """
         Asynchronous function for making HTTP requests, intended to be the most basic web request function
diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns.py b/bbot/test/test_step_2/module_tests/test_module_baddns.py
index dd533669f..5c6d14f45 100644
--- a/bbot/test/test_step_2/module_tests/test_module_baddns.py
+++ b/bbot/test/test_step_2/module_tests/test_module_baddns.py
@@ -31,9 +31,9 @@ async def setup_after_prep(self, module_test):
         module_test.monkeypatch.setattr(WhoisManager, "dispatchWHOIS", self.dispatchWHOIS)
 
     def check(self, module_test, events):
-        assert any(e.data == "baddns.azurewebsites.net" for e in events), "CNAME detection failed"
-        assert any(e.type == "VULNERABILITY" for e in events), "Failed to emit VULNERABILITY"
-        assert any("baddns-cname" in e.tags for e in events), "Failed to add baddns tag"
+        assert any([e.data == "baddns.azurewebsites.net" for e in events]), "CNAME detection failed"
+        assert any([e.type == "VULNERABILITY" for e in events]), "Failed to emit VULNERABILITY"
+        assert any(["baddns-cname" in e.tags for e in events]), "Failed to add baddns tag"
 
 
 class TestBaddns_cname_signature(BaseTestBaddns):
@@ -60,8 +60,8 @@ def set_target(self, target):
         module_test.monkeypatch.setattr(WhoisManager, "dispatchWHOIS", self.dispatchWHOIS)
 
     def check(self, module_test, events):
-        assert any(e for e in events)
+        assert any([e for e in events])
         assert any(
-            e.type == "VULNERABILITY" and "bigcartel.com" in e.data["description"] for e in events
+            [e.type == "VULNERABILITY" and "bigcartel.com" in e.data["description"] for e in events]
         ), "Failed to emit VULNERABILITY"
-        assert any("baddns-cname" in e.tags for e in events), "Failed to add baddns tag"
+        assert any(["baddns-cname" in e.tags for e in events]), "Failed to add baddns tag"

From c52e538a90cdf70119e632e9afb76e8d6c69bc9d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 04:11:44 -0400
Subject: [PATCH 105/220] more testing

---
 bbot/modules/base.py    |  2 +-
 bbot/scanner/manager.py | 33 +++++----------------------------
 2 files changed, 6 insertions(+), 29 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index d054955a3..74e00cae8 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -1488,7 +1488,7 @@ async def queue_outgoing_event(self, event, **kwargs):
         Used by emit_event() to raise new events to the scan
         """
         # if this was a normal module, we'd put it in the outgoing queue
-        # but because it's a intercept module, we need to queue it with the first intercept module
+        # but because it's an intercept module, we need to queue it at the scan's ingress
         await self.scan.ingress_module.queue_event(event, kwargs)
 
     async def queue_event(self, event, kwargs=None):
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index e18120560..6b67e09ea 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -73,7 +73,9 @@ async def handle_event(self, event, kwargs):
             if not event._graph_important:
                 return False, "event was already emitted by its module"
             else:
-                self.debug(f"Event {event} was already emitted by its module, but this time it's graph-important")
+                self.debug(
+                    f"Event {event} was already emitted by its module, but it's graph-important so it gets a pass"
+                )
 
         # update event's scope distance based on its parent
         event.scope_distance = event.source.scope_distance + 1
@@ -178,11 +180,6 @@ def priority(self):
         # we are the lowest priority
         return 99
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # track outgoing duplicates (for `accept_dupes` attribute of modules)
-        self.outgoing_dup_tracker = set()
-
     async def handle_event(self, event, kwargs):
         abort_if = kwargs.pop("abort_if", None)
         on_success_callback = kwargs.pop("on_success_callback", None)
@@ -228,31 +225,11 @@ async def forward_event(self, event, kwargs):
         """
         Queue event with modules
         """
-        is_outgoing_duplicate = self.is_outgoing_duplicate(event)
-        if is_outgoing_duplicate:
-            self.verbose(f"{event.module}: Duplicate event: {event}")
         # absorb event into the word cloud if it's in scope
-        if not is_outgoing_duplicate and -1 < event.scope_distance < 1:
+        if -1 < event.scope_distance < 1:
             self.scan.word_cloud.absorb_event(event)
 
         for mod in self.scan.modules.values():
             # don't distribute events to intercept modules
-            if mod._intercept:
-                continue
-            acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes
-            graph_important = mod._is_graph_important(event)
-            if acceptable_dup or graph_important:
+            if not mod._intercept:
                 await mod.queue_event(event)
-
-    def is_outgoing_duplicate(self, event, add=False):
-        """
-        Calculate whether an event is a duplicate in the context of the whole scan,
-        This will return True if the same event (irregardless of its source module) has been emitted before.
-
-        TODO: Allow modules to use this for custom deduplication such as on a per-host or per-domain basis.
-        """
-        event_hash = hash(event)
-        is_dup = event_hash in self.outgoing_dup_tracker
-        if add:
-            self.outgoing_dup_tracker.add(event_hash)
-        return is_dup

From f4309546dce162c81220b221fbe3586dbbaab23d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 04:34:10 -0400
Subject: [PATCH 106/220] fix bucket tests

---
 bbot/core/helpers/web/engine.py | 8 ++++++--
 bbot/modules/base.py            | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 09f8d9591..263dc9dce 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -7,7 +7,6 @@
 from socksio.exceptions import SOCKSError
 from contextlib import asynccontextmanager
 
-from .client import BBOTAsyncClient
 from bbot.core.engine import EngineServer
 from bbot.core.helpers.misc import bytes_to_human, human_to_bytes, get_exception_chain
 
@@ -34,7 +33,12 @@ def __init__(self, socket_path, target, config={}):
         self.config = config
         self.http_debug = self.config.get("http_debug", False)
         self._ssl_context_noverify = None
-        self.web_client = BBOTAsyncClient.from_config(self.config, self.target, persist_cookies=False)
+        self.web_client = self.AsyncClient(persist_cookies=False)
+
+    def AsyncClient(self, *args, **kwargs):
+        from .client import BBOTAsyncClient
+
+        return BBOTAsyncClient.from_config(self.config, self.target, *args, **kwargs)
 
     async def request(self, *args, **kwargs):
         raise_error = kwargs.pop("raise_error", False)
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 74e00cae8..39594e8bd 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -705,7 +705,7 @@ async def _event_postcheck(self, event):
             # check duplicates
             is_incoming_duplicate, reason = self.is_incoming_duplicate(event, add=True)
             if is_incoming_duplicate and not self.accept_dupes:
-                return False, f"module has already seen {event}" + (f" ({reason})" if reason else "")
+                return False, f"module has already seen it" + (f" ({reason})" if reason else "")
 
         return acceptable, reason
 

From c69afd97c3931db0ecc0bba3f87404c0a779cab3 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 17:15:21 -0400
Subject: [PATCH 107/220] fix tests

---
 bbot/scanner/target.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index 531ae9774..747ff3081 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -31,6 +31,9 @@ def __init__(self, targets, whitelist=None, blacklist=None, strict_scope=False):
             blacklist = []
         self.blacklist = Target(*blacklist)
 
+    def add(self, *args, **kwargs):
+        self.seeds.add(*args, **kwargs)
+
     def merge(self, other):
         self.seeds.add(other.seeds)
         if other.whitelist is not None:

From 96f18d39ae5801f3b8d4f1c7fc43158e1fe89eb3 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 17:49:01 -0400
Subject: [PATCH 108/220] serious debugging

---
 bbot/scanner/manager.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 6b67e09ea..0051aeedb 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -152,15 +152,20 @@ def is_incoming_duplicate(self, event, add=False):
         """
         try:
             event_hash = event.module._outgoing_dedup_hash(event)
+            log.critical(f"{event} native hash == {event_hash}")
         except AttributeError:
             module_name = str(getattr(event, "module", ""))
             event_hash = hash((event, module_name))
+            log.critical(f"{event} fallback hash == {event_hash}")
         is_dup = event_hash in self.incoming_dup_tracker
         if add:
             self.incoming_dup_tracker.add(event_hash)
         suppress_dupes = getattr(event.module, "suppress_dupes", True)
+        log.critical(f"event module {event.module} suppress dupes == {suppress_dupes} and is_dup={is_dup}")
         if suppress_dupes and is_dup:
+            log.critical(f"{event} is dup!!!")
             return True
+        log.critical(f"{event} is NOT dup!#@#@")
         return False
 
 

From faa7bd5a5286d852bb8ad07b2537bf47f449c019 Mon Sep 17 00:00:00 2001
From: BBOT Docs Autopublish <info@blacklanternsecurity.com>
Date: Sat, 11 May 2024 22:27:34 +0000
Subject: [PATCH 109/220] Refresh module docs

---
 README.md                       | 46 ++++++++++++++++-----------------
 docs/modules/list_of_modules.md |  1 +
 docs/scanning/advanced.md       |  2 +-
 docs/scanning/configuration.md  |  2 ++
 docs/scanning/events.md         |  4 +--
 docs/scanning/index.md          | 46 ++++++++++++++++-----------------
 6 files changed, 52 insertions(+), 49 deletions(-)

diff --git a/README.md b/README.md
index f97980608..3fab9b285 100644
--- a/README.md
+++ b/README.md
@@ -264,29 +264,29 @@ BBOT consistently finds 20-50% more subdomains than other tools. The bigger the
 For a full list of modules, including the data types consumed and emitted by each one, see [List of Modules](https://www.blacklanternsecurity.com/bbot/modules/list_of_modules/).
 
 <!-- BBOT MODULE FLAGS -->
-| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| safe             | 80          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
-| passive          | 60          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
-| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                           |
-| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                            |
-| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
-| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| safe             | 81          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
+| passive          | 61          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
+| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                              |
+| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 <!-- END BBOT MODULE FLAGS -->
 
 ## BBOT Output Modules
diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md
index 190b567e9..019b4caaf 100644
--- a/docs/modules/list_of_modules.md
+++ b/docs/modules/list_of_modules.md
@@ -73,6 +73,7 @@
 | git_clone            | scan     | No              | Clone code github repositories                                                                                                          | passive, safe, slow                                                            | CODE_REPOSITORY                                                                                                                         | FILESYSTEM                                                  |
 | github_codesearch    | scan     | Yes             | Query Github's API for code containing the target domain name                                                                           | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | CODE_REPOSITORY, URL_UNVERIFIED                             |
 | github_org           | scan     | No              | Query Github's API for organization and member repositories                                                                             | passive, safe, subdomain-enum                                                  | ORG_STUB, SOCIAL                                                                                                                        | CODE_REPOSITORY                                             |
+| github_workflows     | scan     | No              | Download a github repositories workflow logs                                                                                            | passive, safe                                                                  | CODE_REPOSITORY                                                                                                                         | FILESYSTEM                                                  |
 | hackertarget         | scan     | No              | Query the hackertarget.com API for subdomains                                                                                           | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | hunterio             | scan     | Yes             | Query hunter.io for emails                                                                                                              | email-enum, passive, safe, subdomain-enum                                      | DNS_NAME                                                                                                                                | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED                     |
 | internetdb           | scan     | No              | Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities                                                  | passive, portscan, safe, subdomain-enum                                        | DNS_NAME, IP_ADDRESS                                                                                                                    | DNS_NAME, FINDING, OPEN_TCP_PORT, TECHNOLOGY, VULNERABILITY |
diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md
index b9bef8da7..dbd9cbb37 100644
--- a/docs/scanning/advanced.md
+++ b/docs/scanning/advanced.md
@@ -61,7 +61,7 @@ Target:
 
 Modules:
   -m MODULE [MODULE ...], --modules MODULE [MODULE ...]
-                        Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,baddns,baddns_zone,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,docker_pull,dockerhub,dotnetnuke,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,github_codesearch,github_org,gitlab,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,newsletters,nmap,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,sublist3r,telerik,threatminer,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
+                        Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,baddns,baddns_zone,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,docker_pull,dockerhub,dotnetnuke,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,github_codesearch,github_org,github_workflows,gitlab,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,newsletters,nmap,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,sublist3r,telerik,threatminer,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
   -l, --list-modules    List available modules.
   -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...]
                         Exclude these modules.
diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index 8994138d9..65617e5e2 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -329,6 +329,8 @@ Many modules accept their own configuration options. These options have the abil
 | modules.github_org.api_key                     | str    | Github token                                                                                                                                                                                                                                                                                                                   |                                                                                                                                                                                                                                                                                                                                                                                   |
 | modules.github_org.include_member_repos        | bool   | Also enumerate organization members' repositories                                                                                                                                                                                                                                                                              | False                                                                                                                                                                                                                                                                                                                                                                             |
 | modules.github_org.include_members             | bool   | Enumerate organization members                                                                                                                                                                                                                                                                                                 | True                                                                                                                                                                                                                                                                                                                                                                              |
+| modules.github_workflows.api_key               | str    | Github token                                                                                                                                                                                                                                                                                                                   |                                                                                                                                                                                                                                                                                                                                                                                   |
+| modules.github_workflows.num_logs              | int    | For each workflow fetch the last N successful runs logs (max 100)                                                                                                                                                                                                                                                              | 1                                                                                                                                                                                                                                                                                                                                                                                 |
 | modules.hunterio.api_key                       | str    | Hunter.IO API key                                                                                                                                                                                                                                                                                                              |                                                                                                                                                                                                                                                                                                                                                                                   |
 | modules.ip2location.api_key                    | str    | IP2location.io API Key                                                                                                                                                                                                                                                                                                         |                                                                                                                                                                                                                                                                                                                                                                                   |
 | modules.ip2location.lang                       | str    | Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.                                                                                                                                                                                                            |                                                                                                                                                                                                                                                                                                                                                                                   |
diff --git a/docs/scanning/events.md b/docs/scanning/events.md
index 7f7698c7f..dad28f185 100644
--- a/docs/scanning/events.md
+++ b/docs/scanning/events.md
@@ -54,11 +54,11 @@ Below is a full list of event types along with which modules produce/consume the
 | *                   | 12                    | 0                     | affiliates, csv, discord, http, human, json, neo4j, python, slack, splunk, teams, websocket                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |                                                                                                                                                                                                                                                                                                                                                                                                                                            |
 | ASN                 | 0                     | 1                     |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | asn                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | AZURE_TENANT        | 1                     | 0                     | speculate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| CODE_REPOSITORY     | 2                     | 4                     | docker_pull, git_clone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | dockerhub, github_codesearch, github_org, gitlab                                                                                                                                                                                                                                                                                                                                                                                           |
+| CODE_REPOSITORY     | 3                     | 4                     | docker_pull, git_clone, github_workflows                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | dockerhub, github_codesearch, github_org, gitlab                                                                                                                                                                                                                                                                                                                                                                                           |
 | DNS_NAME            | 57                    | 42                    | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, massdns, myssl, nmap, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomains, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, massdns, myssl, ntlm, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, sublist3r, threatminer, urlscan, vhost, viewdns, virustotal, wayback, zoomeye |
 | DNS_NAME_UNRESOLVED | 3                     | 0                     | baddns, speculate, subdomains                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |                                                                                                                                                                                                                                                                                                                                                                                                                                            |
 | EMAIL_ADDRESS       | 1                     | 6                     | emails                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | credshed, emailformat, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                      |
-| FILESYSTEM          | 1                     | 2                     | trufflehog                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | docker_pull, git_clone                                                                                                                                                                                                                                                                                                                                                                                                                     |
+| FILESYSTEM          | 1                     | 3                     | trufflehog                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | docker_pull, git_clone, github_workflows                                                                                                                                                                                                                                                                                                                                                                                                   |
 | FINDING             | 2                     | 28                    | asset_inventory, web_report                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation                                                                               |
 | GEOLOCATION         | 0                     | 2                     |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | ip2location, ipstack                                                                                                                                                                                                                                                                                                                                                                                                                       |
 | HASHED_PASSWORD     | 0                     | 2                     |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | credshed, dehashed                                                                                                                                                                                                                                                                                                                                                                                                                         |
diff --git a/docs/scanning/index.md b/docs/scanning/index.md
index 0cf499891..588c48b0e 100644
--- a/docs/scanning/index.md
+++ b/docs/scanning/index.md
@@ -107,29 +107,29 @@ A single module can have multiple flags. For example, the `securitytrails` modul
 ### List of Flags
 
 <!-- BBOT MODULE FLAGS -->
-| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| safe             | 80          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
-| passive          | 60          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
-| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                           |
-| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                            |
-| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
-| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
-| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| safe             | 81          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
+| passive          | 61          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
+| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                              |
+| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
 <!-- END BBOT MODULE FLAGS --> 
 
 ## Dependencies

From 0207dbc07169484eddbc7cd80bd934808b2563fa Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 21:38:40 -0400
Subject: [PATCH 110/220] gotcha

---
 bbot/modules/internal/dns.py                        | 8 ++++----
 bbot/scanner/manager.py                             | 5 -----
 bbot/scanner/scanner.py                             | 1 +
 bbot/test/test_step_1/test_manager_deduplication.py | 2 --
 4 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index f501fb08f..db1979d3a 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -54,14 +54,12 @@ async def handle_event(self, event, kwargs):
         dns_children = dict()
         event_whitelisted = False
         event_blacklisted = False
+        emit_children = False
 
         event_host = str(event.host)
         event_host_hash = hash(str(event.host))
         event_is_ip = self.helpers.is_ip(event.host)
 
-        # only emit DNS children if we haven't seen this host before
-        emit_children = self.dns_resolution and event_host_hash not in self._event_cache
-
         # we do DNS resolution inside a lock to make sure we don't duplicate work
         # once the resolution happens, it will be cached so it doesn't need to happen again
         async with self._event_cache_locks.lock(event_host_hash):
@@ -123,6 +121,9 @@ async def handle_event(self, event, kwargs):
                         except ValueError:
                             continue
 
+                # only emit DNS children if we haven't seen this host before
+                emit_children = self.dns_resolution and event_host_hash not in self._event_cache
+
                 # store results in cache
                 self._event_cache[event_host_hash] = dns_tags, dns_children, event_whitelisted, event_blacklisted
 
@@ -189,7 +190,6 @@ async def handle_event(self, event, kwargs):
             if dns_children:
                 for rdtype, records in dns_children.items():
                     module = self.scan._make_dummy_module_dns(rdtype)
-                    module._priority = 4
                     for record in records:
                         try:
                             child_event = self.scan.make_event(record, "DNS_NAME", module=module, source=source_event)
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 0051aeedb..6b67e09ea 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -152,20 +152,15 @@ def is_incoming_duplicate(self, event, add=False):
         """
         try:
             event_hash = event.module._outgoing_dedup_hash(event)
-            log.critical(f"{event} native hash == {event_hash}")
         except AttributeError:
             module_name = str(getattr(event, "module", ""))
             event_hash = hash((event, module_name))
-            log.critical(f"{event} fallback hash == {event_hash}")
         is_dup = event_hash in self.incoming_dup_tracker
         if add:
             self.incoming_dup_tracker.add(event_hash)
         suppress_dupes = getattr(event.module, "suppress_dupes", True)
-        log.critical(f"event module {event.module} suppress dupes == {suppress_dupes} and is_dup={is_dup}")
         if suppress_dupes and is_dup:
-            log.critical(f"{event} is dup!!!")
             return True
-        log.critical(f"{event} is NOT dup!#@#@")
         return False
 
 
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index ee17ef6a7..308d58bdf 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -1164,6 +1164,7 @@ def _make_dummy_module_dns(self, name):
         except KeyError:
             dummy_module = self._make_dummy_module(name=name, _type="DNS")
             dummy_module.suppress_dupes = False
+            dummy_module._priority = 4
             self.dummy_modules[name] = dummy_module
         return dummy_module
 
diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 3dea17f3b..3f88576f9 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -117,8 +117,6 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
 
-    for e in all_events:
-        log.critical(f"{e}:{e._graph_important} <-- {e.source}:{e.source._graph_important} <-- {e.source.source}:{e.source.source._graph_important}")
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])

From 80587cd6a1794460db59de97ba902c72bb2b7800 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sat, 11 May 2024 22:52:25 -0400
Subject: [PATCH 111/220] fix ffuf shortnames tests

---
 bbot/modules/internal/cloud.py                 | 18 ++++++++----------
 .../test_module_ffuf_shortnames.py             |  3 +--
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py
index ac4eb71f7..2b519fbe5 100644
--- a/bbot/modules/internal/cloud.py
+++ b/bbot/modules/internal/cloud.py
@@ -8,18 +8,13 @@ class cloud(InterceptModule):
     _priority = 3
 
     async def setup(self):
-        self._dummy_modules = None
+        self.dummy_modules = None
         return True
 
-    @property
-    def dummy_modules(self):
-        if self._dummy_modules is None:
-            self._dummy_modules = {}
-            for provider_name, provider in self.helpers.cloud.providers.items():
-                self._dummy_modules[provider_name] = self.scan._make_dummy_module(
-                    f"cloud_{provider_name}", _type="scan"
-                )
-        return self._dummy_modules
+    def get_dummy_modules(self):
+        self.dummy_modules = {}
+        for provider_name, provider in self.helpers.cloud.providers.items():
+            self.dummy_modules[provider_name] = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan")
 
     async def filter_event(self, event):
         if (not event.host) or (event.type in ("IP_RANGE",)):
@@ -27,6 +22,9 @@ async def filter_event(self, event):
         return True
 
     async def handle_event(self, event, kwargs):
+        # don't hold up the event loop loading cloud IPs etc.
+        if self.dummy_modules is None:
+            await self.helpers.run_in_executor(self.get_dummy_modules)
         # cloud tagging by hosts
         hosts_to_check = set(str(s) for s in event.resolved_hosts)
         hosts_to_check.add(str(event.host_original))
diff --git a/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py b/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py
index 1f624a410..00c1f9b1e 100644
--- a/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py
+++ b/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py
@@ -7,7 +7,6 @@ class TestFFUFShortnames(ModuleTestBase):
     config_overrides = {
         "modules": {
             "ffuf_shortnames": {
-                "find_common_prefixes": True,
                 "find_common_prefixes": True,
                 "wordlist": tempwordlist(test_wordlist),
             }
@@ -143,7 +142,7 @@ async def setup_after_prep(self, module_test):
                 tags=["shortname-file"],
             )
         )
-        module_test.scan.target._events = set(seed_events)
+        module_test.scan.target.seeds._events = set(seed_events)
 
         expect_args = {"method": "GET", "uri": "/administrator.aspx"}
         respond_args = {"response_data": "alive"}

From af9927383cb944f2725037273a22ea1076c96fd6 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 12 May 2024 00:10:34 -0400
Subject: [PATCH 112/220] better error handling

---
 bbot/core/engine.py                  | 9 ++++-----
 bbot/test/test_step_1/test_dns.py    | 3 +++
 bbot/test/test_step_1/test_engine.py | 5 ++---
 bbot/test/test_step_1/test_web.py    | 7 ++++++-
 4 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 412a5b26d..18e48e7d4 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -46,9 +46,8 @@ def check_error(self, message):
         if message is error_sentinel:
             return True
         if isinstance(message, dict) and len(message) == 1 and "_e" in message:
-            error = message["_e"][0]
-            raise BBOTEngineError(error)
-            return True
+            error = message["_e"]
+            raise error
         return False
 
 
@@ -211,7 +210,7 @@ async def run_and_return(self, client_id, command_fn, *args, **kwargs):
             trace = traceback.format_exc()
             self.log.error(error)
             self.log.trace(trace)
-            result = {"_e": (error, trace)}
+            result = {"_e": e}
         finally:
             self.tasks.pop(client_id, None)
         await self.send_socket_multipart([client_id, pickle.dumps(result)])
@@ -227,7 +226,7 @@ async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
             trace = traceback.format_exc()
             self.log.error(error)
             self.log.trace(trace)
-            result = {"_e": (error, trace)}
+            result = {"_e": e}
             await self.send_socket_multipart([client_id, pickle.dumps(result)])
         finally:
             self.tasks.pop(client_id, None)
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 0e8ca141b..1fe8d05cc 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -4,6 +4,9 @@
 @pytest.mark.asyncio
 async def test_dns_engine(bbot_scanner):
     scan = bbot_scanner()
+    await scan.helpers._mock_dns(
+        {"one.one.one.one": {"A": ["1.1.1.1"]}, "1.1.1.1.in-addr.arpa": {"PTR": ["one.one.one.one"]}}
+    )
     result = await scan.helpers.resolve("one.one.one.one")
     assert "1.1.1.1" in result
     assert not "2606:4700:4700::1111" in result
diff --git a/bbot/test/test_step_1/test_engine.py b/bbot/test/test_step_1/test_engine.py
index ff0f85174..f24ea59c0 100644
--- a/bbot/test/test_step_1/test_engine.py
+++ b/bbot/test/test_step_1/test_engine.py
@@ -3,7 +3,6 @@
 
 @pytest.mark.asyncio
 async def test_engine():
-    from bbot.errors import BBOTEngineError
     from bbot.core.engine import EngineClient, EngineServer
 
     counter = 0
@@ -97,7 +96,7 @@ async def yield_stuff(self, n):
     yield_cancelled = False
     yield_errored = False
     agen = test_engine.yield_stuff(None)
-    with pytest.raises(BBOTEngineError):
+    with pytest.raises(TypeError):
         async for _ in agen:
             pass
     assert yield_cancelled == False
@@ -139,7 +138,7 @@ async def yield_stuff(self, n):
     return_finished = False
     return_cancelled = False
     return_errored = False
-    with pytest.raises(BBOTEngineError):
+    with pytest.raises(TypeError):
         result = await test_engine.return_thing(None)
     assert return_started == True
     assert return_finished == False
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index 2ccf06e32..21fefe5d6 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -1,10 +1,11 @@
 import re
+import httpx
 
 from ..bbot_fixtures import *
 
 
 @pytest.mark.asyncio
-async def test_web_engine(bbot_scanner, bbot_httpserver):
+async def test_web_engine(bbot_scanner, bbot_httpserver, httpx_mock):
 
     web_body = "hello_there" * 1000
     url = bbot_httpserver.url_for("/test")
@@ -39,6 +40,10 @@ async def test_web_engine(bbot_scanner, bbot_httpserver):
     file_content = open(filename).read()
     assert file_content == web_body
 
+    # raise_error=True
+    with pytest.raises(httpx.TimeoutException):
+        await scan.helpers.request("http://www.example.com/", raise_error=True)
+
 
 @pytest.mark.asyncio
 async def test_web_helpers(bbot_scanner, bbot_httpserver):

From 1aa0f6ece0318f61655d9e4b05537db181f85825 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 12 May 2024 00:14:10 -0400
Subject: [PATCH 113/220] delayed initialization for asyncio.lock

---
 bbot/core/helpers/dns/engine.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py
index 6018e0e3f..62e8cb201 100644
--- a/bbot/core/helpers/dns/engine.py
+++ b/bbot/core/helpers/dns/engine.py
@@ -80,7 +80,7 @@ def __init__(self, socket_path, config={}):
         # modules from kicking off wildcard detection for the same domain at the same time
         self._wildcard_lock = NamedLock()
 
-        self._dns_connectivity_lock = asyncio.Lock()
+        self._dns_connectivity_lock = None
         self._last_dns_success = None
         self._last_connectivity_warning = time.time()
         # keeps track of warnings issued for wildcard detection to prevent duplicate warnings
@@ -669,6 +669,12 @@ async def is_wildcard_domain(self, domain, log_info=False):
 
         return wildcard_domain_results
 
+    @property
+    def dns_connectivity_lock(self):
+        if self._dns_connectivity_lock is None:
+            self._dns_connectivity_lock = asyncio.Lock()
+        return self._dns_connectivity_lock
+
     async def _connectivity_check(self, interval=5):
         """
         Periodically checks for an active internet connection by attempting DNS resolution.
@@ -688,7 +694,7 @@ async def _connectivity_check(self, interval=5):
             if time.time() - self._last_dns_success < interval:
                 return True
         dns_server_working = []
-        async with self._dns_connectivity_lock:
+        async with self.dns_connectivity_lock:
             with suppress(Exception):
                 dns_server_working = await self._catch(self.resolver.resolve, "www.google.com", rdtype="A")
                 if dns_server_working:

From 3bce8369f9316b042429cc9c5e9700b129f562e8 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Sun, 12 May 2024 10:41:16 +0100
Subject: [PATCH 114/220] Ensure duplicated secret in zip folder does not get
 detected

---
 bbot/test/test_step_2/module_tests/test_module_trufflehog.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
index 827f76d6f..4d7122276 100644
--- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
+++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
@@ -541,6 +541,7 @@ async def setup_before_prep(self, module_test):
         data = io.BytesIO()
         with zipfile.ZipFile(data, mode="w", compression=zipfile.ZIP_DEFLATED) as z:
             z.writestr("test.txt", self.file_content)
+            z.writestr("folder/test2.txt", self.file_content)
         data.seek(0)
         zip_content = data.getvalue()
         module_test.httpx_mock.add_response(
@@ -861,7 +862,7 @@ def check(self, module_test, events):
             assert content == self.file_content, "File content doesn't match"
         github_workflow_event = [e for e in vuln_events if "bbot" in e.data["description"]][0].source
         file = Path(github_workflow_event.data["path"])
-        assert file.is_file(), "Destination zip does not exist"
+        assert file.is_file(), "Destination file does not exist"
         docker_source_event = [e for e in vuln_events if e.data["host"] == "hub.docker.com"][0].source
         file = Path(docker_source_event.data["path"])
         assert file.is_file(), "Destination image does not exist"
@@ -888,7 +889,7 @@ def check(self, module_test, events):
             assert content == self.file_content, "File content doesn't match"
         github_workflow_event = [e for e in finding_events if "bbot" in e.data["description"]][0].source
         file = Path(github_workflow_event.data["path"])
-        assert file.is_file(), "Destination zip does not exist"
+        assert file.is_file(), "Destination file does not exist"
         docker_source_event = [e for e in finding_events if e.data["host"] == "hub.docker.com"][0].source
         file = Path(docker_source_event.data["path"])
         assert file.is_file(), "Destination image does not exist"

From 97829e61c47b06d447cfcd0f79d302390d1eede8 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Sun, 12 May 2024 10:41:43 +0100
Subject: [PATCH 115/220] Update trufflehog version AGAIN

---
 bbot/modules/trufflehog.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index 0ec607457..5acb45a12 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -9,7 +9,7 @@ class trufflehog(BaseModule):
     meta = {"description": "TruffleHog is a tool for finding credentials"}
 
     options = {
-        "version": "3.74.0",
+        "version": "3.75.1",
         "only_verified": True,
         "concurrency": 8,
     }

From 30f72d7f81f1e0a0c2d14a97516aa53895f7f48f Mon Sep 17 00:00:00 2001
From: BBOT Docs Autopublish <info@blacklanternsecurity.com>
Date: Sun, 12 May 2024 13:53:23 +0000
Subject: [PATCH 116/220] Refresh module docs

---
 docs/scanning/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index 65617e5e2..3a02bcc9a 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -348,7 +348,7 @@ Many modules accept their own configuration options. These options have the abil
 | modules.shodan_dns.api_key                     | str    | Shodan API key                                                                                                                                                                                                                                                                                                                 |                                                                                                                                                                                                                                                                                                                                                                                   |
 | modules.trufflehog.concurrency                 | int    | Number of concurrent workers                                                                                                                                                                                                                                                                                                   | 8                                                                                                                                                                                                                                                                                                                                                                                 |
 | modules.trufflehog.only_verified               | bool   | Only report credentials that have been verified                                                                                                                                                                                                                                                                                | True                                                                                                                                                                                                                                                                                                                                                                              |
-| modules.trufflehog.version                     | str    | trufflehog version                                                                                                                                                                                                                                                                                                             | 3.69.0                                                                                                                                                                                                                                                                                                                                                                            |
+| modules.trufflehog.version                     | str    | trufflehog version                                                                                                                                                                                                                                                                                                             | 3.75.1                                                                                                                                                                                                                                                                                                                                                                            |
 | modules.urlscan.urls                           | bool   | Emit URLs in addition to DNS_NAMEs                                                                                                                                                                                                                                                                                             | False                                                                                                                                                                                                                                                                                                                                                                             |
 | modules.virustotal.api_key                     | str    | VirusTotal API Key                                                                                                                                                                                                                                                                                                             |                                                                                                                                                                                                                                                                                                                                                                                   |
 | modules.wayback.garbage_threshold              | int    | Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data)                                                                                                                                                                                                                          | 10                                                                                                                                                                                                                                                                                                                                                                                |

From 8eddffadafa5aefcb62b5f13b49320fd023fbf27 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 12 May 2024 12:22:11 -0400
Subject: [PATCH 117/220] fix tests

---
 bbot/test/test_step_1/test_dns.py | 79 ++++++++++++++++---------------
 1 file changed, 41 insertions(+), 38 deletions(-)

diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 1fe8d05cc..4dac8b71f 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -238,29 +238,31 @@ async def test_wildcards(bbot_scanner):
         e for e in events if e.type == "DNS_NAME" and e.data == "asdfl.gashdgkjsadgsdf.github.io"
     ]
     assert len(unmodified_wildcard_events) == 1
-    assert unmodified_wildcard_events[0].tags == {
-        "a-record",
-        "target",
-        "aaaa-wildcard",
-        "in-scope",
-        "subdomain",
-        "cdn-github",
-        "aaaa-record",
-        "wildcard",
-        "a-wildcard",
-    }
+    assert unmodified_wildcard_events[0].tags.issuperset(
+        {
+            "a-record",
+            "target",
+            "aaaa-wildcard",
+            "in-scope",
+            "subdomain",
+            "aaaa-record",
+            "wildcard",
+            "a-wildcard",
+        }
+    )
     modified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and e.data == "_wildcard.github.io"]
     assert len(modified_wildcard_events) == 1
-    assert modified_wildcard_events[0].tags == {
-        "a-record",
-        "aaaa-wildcard",
-        "in-scope",
-        "subdomain",
-        "cdn-github",
-        "aaaa-record",
-        "wildcard",
-        "a-wildcard",
-    }
+    assert modified_wildcard_events[0].tags.issuperset(
+        {
+            "a-record",
+            "aaaa-wildcard",
+            "in-scope",
+            "subdomain",
+            "aaaa-record",
+            "wildcard",
+            "a-wildcard",
+        }
+    )
     assert modified_wildcard_events[0].host_original == "lkjg.sdfgsg.jgkhajshdsadf.github.io"
 
     # test with full scan (wildcard detection disabled for domain)
@@ -268,6 +270,7 @@ async def test_wildcards(bbot_scanner):
         "asdfl.gashdgkjsadgsdf.github.io",
         whitelist=["github.io"],
         config={"dns_wildcard_ignore": ["github.io"], "dns_resolution": True},
+        exclude_modules=["cloud"],
     )
     await scan2._prep()
     other_event = scan2.make_event(
@@ -284,15 +287,15 @@ async def test_wildcards(bbot_scanner):
             e
             for e in unmodified_wildcard_events
             if e.data == "asdfl.gashdgkjsadgsdf.github.io"
-            and e.tags
-            == {
-                "target",
-                "a-record",
-                "in-scope",
-                "subdomain",
-                "cdn-github",
-                "aaaa-record",
-            }
+            and e.tags.issuperset(
+                {
+                    "target",
+                    "a-record",
+                    "in-scope",
+                    "subdomain",
+                    "aaaa-record",
+                }
+            )
         ]
     )
     assert 1 == len(
@@ -300,14 +303,14 @@ async def test_wildcards(bbot_scanner):
             e
             for e in unmodified_wildcard_events
             if e.data == "lkjg.sdfgsg.jgkhajshdsadf.github.io"
-            and e.tags
-            == {
-                "a-record",
-                "in-scope",
-                "subdomain",
-                "cdn-github",
-                "aaaa-record",
-            }
+            and e.tags.issuperset(
+                {
+                    "a-record",
+                    "in-scope",
+                    "subdomain",
+                    "aaaa-record",
+                }
+            )
         ]
     )
     modified_wildcard_events = [e for e in events if e.type == "DNS_NAME" and e.data == "_wildcard.github.io"]

From f67b259169ba197804b32741fc70066d297beabf Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 12 May 2024 12:54:18 -0400
Subject: [PATCH 118/220] fix minor race condition

---
 bbot/modules/internal/cloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py
index 2b519fbe5..0231f6155 100644
--- a/bbot/modules/internal/cloud.py
+++ b/bbot/modules/internal/cloud.py
@@ -12,7 +12,6 @@ async def setup(self):
         return True
 
     def get_dummy_modules(self):
-        self.dummy_modules = {}
         for provider_name, provider in self.helpers.cloud.providers.items():
             self.dummy_modules[provider_name] = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan")
 
@@ -24,6 +23,7 @@ async def filter_event(self, event):
     async def handle_event(self, event, kwargs):
         # don't hold up the event loop loading cloud IPs etc.
         if self.dummy_modules is None:
+            self.dummy_modules = {}
             await self.helpers.run_in_executor(self.get_dummy_modules)
         # cloud tagging by hosts
         hosts_to_check = set(str(s) for s in event.resolved_hosts)

From 89984cca6c46653e81f169507f89dafb26f86d33 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 12 May 2024 13:38:18 -0400
Subject: [PATCH 119/220] working on tests

---
 bbot/modules/internal/cloud.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py
index 0231f6155..27c56bc0d 100644
--- a/bbot/modules/internal/cloud.py
+++ b/bbot/modules/internal/cloud.py
@@ -11,7 +11,8 @@ async def setup(self):
         self.dummy_modules = None
         return True
 
-    def get_dummy_modules(self):
+    def make_dummy_modules(self):
+        self.dummy_modules = {}
         for provider_name, provider in self.helpers.cloud.providers.items():
             self.dummy_modules[provider_name] = self.scan._make_dummy_module(f"cloud_{provider_name}", _type="scan")
 
@@ -23,8 +24,7 @@ async def filter_event(self, event):
     async def handle_event(self, event, kwargs):
         # don't hold up the event loop loading cloud IPs etc.
         if self.dummy_modules is None:
-            self.dummy_modules = {}
-            await self.helpers.run_in_executor(self.get_dummy_modules)
+            self.make_dummy_modules()
         # cloud tagging by hosts
         hosts_to_check = set(str(s) for s in event.resolved_hosts)
         hosts_to_check.add(str(event.host_original))

From 50b1cf4e8c8aa96cc82a864e617a90d90e269d88 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 12 May 2024 22:21:36 -0400
Subject: [PATCH 120/220] code comments

---
 bbot/core/helpers/web/web.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 5cf7141fb..819b5d6bf 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -106,10 +106,44 @@ async def request(self, *args, **kwargs):
         return await self.run_and_return("request", *args, **kwargs)
 
     async def request_batch(self, urls, *args, **kwargs):
+        """
+        Given a list of URLs, request them in parallel and yield responses as they come in.
+
+        Args:
+            urls (list[str]): List of URLs to visit
+            *args: Positional arguments to pass through to httpx
+            **kwargs: Keyword arguments to pass through to httpx
+
+        Examples:
+            >>> async for url, response in self.helpers.request_batch(urls, headers={"X-Test": "Test"}):
+            >>>     if response is not None and response.status_code == 200:
+            >>>         self.hugesuccess(response)
+        """
         async for _ in self.run_and_yield("request_batch", urls, *args, **kwargs):
             yield _
 
     async def request_custom_batch(self, urls_and_args):
+        """
+        Make web requests in parallel with custom options for each request. Yield responses as they come in.
+
+        Similar to `request_batch` except it allows individual arguments for each URL.
+
+        Args:
+            urls_and_kwargs (list[tuple]): List of tuples in the format: (url, kwargs, custom_tracker)
+                where custom_tracker is an optional value for your own internal use. You may use it to
+                help correlate requests, etc.
+
+        Examples:
+            >>> urls_and_kwargs = [
+            >>>     ("http://evilcorp.com/1", {"method": "GET"}, "request-1"),
+            >>>     ("http://evilcorp.com/2", {"method": "POST"}, "request-2"),
+            >>> ]
+            >>> async for url, kwargs, custom_tracker, response in self.helpers.request_custom_batch(
+            >>>     urls_and_kwargs
+            >>> ):
+            >>>     if response is not None and response.status_code == 200:
+            >>>         self.hugesuccess(response)
+        """
         async for _ in self.run_and_yield("request_custom_batch", urls_and_args):
             yield _
 

From 630c1ed51e1fe02b0a29ab96be8bd5d2c7656eaa Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 May 2024 04:32:20 +0000
Subject: [PATCH 121/220] Bump pre-commit from 3.7.0 to 3.7.1

Bumps [pre-commit](https://github.com/pre-commit/pre-commit) from 3.7.0 to 3.7.1.
- [Release notes](https://github.com/pre-commit/pre-commit/releases)
- [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md)
- [Commits](https://github.com/pre-commit/pre-commit/compare/v3.7.0...v3.7.1)

---
updated-dependencies:
- dependency-name: pre-commit
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 1f6d2500e..2132bde85 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1444,13 +1444,13 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"]
 
 [[package]]
 name = "pre-commit"
-version = "3.7.0"
+version = "3.7.1"
 description = "A framework for managing and maintaining multi-language pre-commit hooks."
 optional = false
 python-versions = ">=3.9"
 files = [
-    {file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"},
-    {file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"},
+    {file = "pre_commit-3.7.1-py2.py3-none-any.whl", hash = "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5"},
+    {file = "pre_commit-3.7.1.tar.gz", hash = "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a"},
 ]
 
 [package.dependencies]

From dbfa02e598f0a61c932eb96e50a0b94b048ce30a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 10:12:35 -0400
Subject: [PATCH 122/220] fix subdomain enum duplicate query bug

---
 bbot/modules/templates/subdomain_enum.py      |  6 ++
 bbot/test/test_step_2/module_tests/base.py    |  4 +-
 .../test_step_2/template_tests/__init__.py    |  0
 .../test_template_subdomain_enum.py           | 85 +++++++++++++++++++
 4 files changed, 93 insertions(+), 2 deletions(-)
 create mode 100644 bbot/test/test_step_2/template_tests/__init__.py
 create mode 100644 bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py

diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 18243c393..65c0cd6d3 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -30,6 +30,12 @@ class subdomain_enum(BaseModule):
     #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
     dedup_strategy = "highest_parent"
 
+    def _incoming_dedup_hash(self, event):
+        """
+        Determines the criteria for what is considered to be a duplicate event if `accept_dupes` is False.
+        """
+        return hash(self.make_query(event)), f"dedup_strategy={self.dedup_strategy}"
+
     async def handle_event(self, event):
         query = self.make_query(event)
         results = await self.query(query)
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 252e58641..717fde226 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -43,7 +43,7 @@ class ModuleTestBase:
     whitelist = None
     module_name = None
     config_overrides = {}
-    modules_overrides = []
+    modules_overrides = None
     log = logging.getLogger("bbot")
 
     class ModuleTest:
@@ -147,7 +147,7 @@ def _scan_name(self):
 
     @property
     def modules(self):
-        if self.modules_overrides:
+        if self.modules_overrides is not None:
             return self.modules_overrides
         return [self.name]
 
diff --git a/bbot/test/test_step_2/template_tests/__init__.py b/bbot/test/test_step_2/template_tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
new file mode 100644
index 000000000..026895108
--- /dev/null
+++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
@@ -0,0 +1,85 @@
+from ..module_tests.base import ModuleTestBase
+
+
+class TestSubdomainEnum(ModuleTestBase):
+    targets = ["blacklanternsecurity.com"]
+    modules_overrides = []
+    config_overrides = {"dns_resolution": True, "scope_report_distance": 10}
+    dedup_strategy = "highest_parent"
+
+    txt = [
+        "www.blacklanternsecurity.com",
+        "asdf.www.blacklanternsecurity.com",
+        "test.asdf.www.blacklanternsecurity.com",
+        "api.test.asdf.www.blacklanternsecurity.com",
+    ]
+
+    async def setup_after_prep(self, module_test):
+        dns_mock = {
+            "evilcorp.com": {"A": ["127.0.0.6"]},
+            "blacklanternsecurity.com": {"A": ["127.0.0.5"]},
+            "www.blacklanternsecurity.com": {"A": ["127.0.0.5"]},
+            "asdf.www.blacklanternsecurity.com": {"A": ["127.0.0.5"]},
+            "test.asdf.www.blacklanternsecurity.com": {"A": ["127.0.0.5"]},
+            "api.test.asdf.www.blacklanternsecurity.com": {"A": ["127.0.0.5"]},
+        }
+        if self.txt:
+            dns_mock["blacklanternsecurity.com"]["TXT"] = self.txt
+        await module_test.mock_dns(dns_mock)
+
+        # load subdomain enum template as module
+        from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+        subdomain_enum_module = subdomain_enum(module_test.scan)
+
+        self.queries = []
+
+        async def mock_query(query):
+            self.queries.append(query)
+
+        subdomain_enum_module.query = mock_query
+        subdomain_enum_module.dedup_strategy = self.dedup_strategy
+        module_test.scan.modules["subdomain_enum"] = subdomain_enum_module
+
+    def check(self, module_test, events):
+        in_scope_dns_names = [e for e in events if e.type == "DNS_NAME" and e.scope_distance == 0]
+        assert len(in_scope_dns_names) == 5
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "www.blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "asdf.www.blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "test.asdf.www.blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "api.test.asdf.www.blacklanternsecurity.com"])
+        assert len(self.queries) == 1
+        assert self.queries[0] == "blacklanternsecurity.com"
+
+
+class TestSubdomainEnumHighestParent(TestSubdomainEnum):
+    targets = ["api.test.asdf.www.blacklanternsecurity.com", "evilcorp.com"]
+    whitelist = ["www.blacklanternsecurity.com"]
+    modules_overrides = ["speculate"]
+    dedup_strategy = "highest_parent"
+    txt = None
+
+    def check(self, module_test, events):
+        in_scope_dns_names = [e for e in events if e.type == "DNS_NAME" and e.scope_distance == 0]
+        distance_1_dns_names = [e for e in events if e.type == "DNS_NAME" and e.scope_distance == 1]
+        assert len(in_scope_dns_names) == 4
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "www.blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "asdf.www.blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "test.asdf.www.blacklanternsecurity.com"])
+        assert 1 == len([e for e in in_scope_dns_names if e.data == "api.test.asdf.www.blacklanternsecurity.com"])
+        assert len(distance_1_dns_names) == 1
+        assert 1 == len([e for e in distance_1_dns_names if e.data == "evilcorp.com"])
+        assert len(self.queries) == 1
+        assert self.queries[0] == "www.blacklanternsecurity.com"
+
+
+class TestSubdomainEnumLowestParent(TestSubdomainEnumHighestParent):
+    dedup_strategy = "lowest_parent"
+
+    def check(self, module_test, events):
+        assert set(self.queries) == {
+            "test.asdf.www.blacklanternsecurity.com",
+            "asdf.www.blacklanternsecurity.com",
+            "www.blacklanternsecurity.com",
+        }

From 604562abf9e011b30b5ab1de45ef46ce0b20d931 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 14:13:01 -0400
Subject: [PATCH 123/220] better engine error handling

---
 bbot/core/engine.py                      | 84 +++++++++++++++---------
 bbot/core/helpers/dns/dns.py             |  2 +
 bbot/core/helpers/misc.py                |  7 +-
 bbot/core/helpers/web/engine.py          | 59 ++++++++++-------
 bbot/core/helpers/web/web.py             | 58 +++++++++++++++-
 bbot/errors.py                           |  8 +++
 bbot/modules/templates/subdomain_enum.py |  3 +-
 bbot/test/test_step_1/test_helpers.py    |  4 ++
 bbot/test/test_step_1/test_web.py        | 24 ++++++-
 9 files changed, 186 insertions(+), 63 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 18e48e7d4..de09cc93b 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -22,6 +22,9 @@
 
 
 class EngineBase:
+
+    ERROR_CLASS = BBOTEngineError
+
     def __init__(self):
         self.log = logging.getLogger(f"bbot.core.{self.__class__.__name__.lower()}")
 
@@ -42,14 +45,6 @@ def unpickle(self, binary):
             self.log.trace(traceback.format_exc())
         return error_sentinel
 
-    def check_error(self, message):
-        if message is error_sentinel:
-            return True
-        if isinstance(message, dict) and len(message) == 1 and "_e" in message:
-            error = message["_e"]
-            raise error
-        return False
-
 
 class EngineClient(EngineBase):
 
@@ -72,6 +67,14 @@ def __init__(self, **kwargs):
         self.context = zmq.asyncio.Context()
         atexit.register(self.cleanup)
 
+    def check_error(self, message):
+        if isinstance(message, dict) and len(message) == 1 and "_e" in message:
+            error, trace = message["_e"]
+            error = self.ERROR_CLASS(error)
+            error.engine_traceback = trace
+            raise error
+        return False
+
     async def run_and_return(self, command, *args, **kwargs):
         async with self.new_socket() as socket:
             try:
@@ -202,42 +205,59 @@ def __init__(self, socket_path):
             self.tasks = dict()
 
     async def run_and_return(self, client_id, command_fn, *args, **kwargs):
-        self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({args}, {kwargs})")
         try:
-            result = await command_fn(*args, **kwargs)
+            self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({args}, {kwargs})")
+            try:
+                result = await command_fn(*args, **kwargs)
+            except BaseException as e:
+                error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
+                trace = traceback.format_exc()
+                self.log.error(error)
+                self.log.trace(trace)
+                result = {"_e": (error, trace)}
+            finally:
+                self.tasks.pop(client_id, None)
+            await self.send_socket_multipart(client_id, result)
         except BaseException as e:
-            error = f"Unhandled error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
-            trace = traceback.format_exc()
-            self.log.error(error)
-            self.log.trace(trace)
-            result = {"_e": e}
-        finally:
-            self.tasks.pop(client_id, None)
-        await self.send_socket_multipart([client_id, pickle.dumps(result)])
+            self.log.critical(
+                f"Unhandled exception in {self.name}.run_and_return({client_id}, {command_fn}, {args}, {kwargs}): {e}"
+            )
+            self.log.critical(traceback.format_exc())
 
     async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
-        self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({args}, {kwargs})")
         try:
-            async for _ in command_fn(*args, **kwargs):
-                await self.send_socket_multipart([client_id, pickle.dumps(_)])
-            await self.send_socket_multipart([client_id, pickle.dumps({"_s": None})])
+            self.log.debug(f"{self.name} run-and-yield {command_fn.__name__}({args}, {kwargs})")
+            try:
+                async for _ in command_fn(*args, **kwargs):
+                    await self.send_socket_multipart(client_id, _)
+                await self.send_socket_multipart(client_id, {"_s": None})
+            except BaseException as e:
+                error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
+                trace = traceback.format_exc()
+                self.log.error(error)
+                self.log.trace(trace)
+                result = {"_e": (error, trace)}
+                await self.send_socket_multipart(client_id, result)
+            finally:
+                self.tasks.pop(client_id, None)
         except BaseException as e:
-            error = f"Unhandled error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
-            trace = traceback.format_exc()
-            self.log.error(error)
-            self.log.trace(trace)
-            result = {"_e": e}
-            await self.send_socket_multipart([client_id, pickle.dumps(result)])
-        finally:
-            self.tasks.pop(client_id, None)
+            self.log.critical(
+                f"Unhandled exception in {self.name}.run_and_yield({client_id}, {command_fn}, {args}, {kwargs}): {e}"
+            )
+            self.log.critical(traceback.format_exc())
 
-    async def send_socket_multipart(self, *args, **kwargs):
+    async def send_socket_multipart(self, client_id, message):
         try:
-            await self.socket.send_multipart(*args, **kwargs)
+            message = pickle.dumps(message)
+            await self.socket.send_multipart([client_id, message])
         except Exception as e:
             self.log.warning(f"Error sending ZMQ message: {e}")
             self.log.trace(traceback.format_exc())
 
+    def check_error(self, message):
+        if message is error_sentinel:
+            return True
+
     async def worker(self):
         try:
             while 1:
diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py
index a4d626e5c..aec01126d 100644
--- a/bbot/core/helpers/dns/dns.py
+++ b/bbot/core/helpers/dns/dns.py
@@ -4,6 +4,7 @@
 import dns.asyncresolver
 from radixtarget import RadixTarget
 
+from bbot.errors import DNSError
 from bbot.core.engine import EngineClient
 from ..misc import clean_dns_record, is_ip, is_domain, is_dns_name
 
@@ -15,6 +16,7 @@
 class DNSHelper(EngineClient):
 
     SERVER_CLASS = DNSEngine
+    ERROR_CLASS = DNSError
 
     """Helper class for DNS-related operations within BBOT.
 
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 0d1b6be06..09d2870d4 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -236,6 +236,7 @@ def split_host_port(d):
 
     port = match.group(3)
     if port is None and scheme is not None:
+        scheme = scheme.lower()
         if scheme in ("https", "wss"):
             port = 443
         elif scheme in ("http", "ws"):
@@ -2115,7 +2116,7 @@ def os_platform_friendly():
 tag_filter_regex = re.compile(r"[^a-z0-9]+")
 
 
-def tagify(s, maxlen=None):
+def tagify(s, delimiter=None, maxlen=None):
     """Sanitize a string into a tag-friendly format.
 
     Converts a given string to lowercase and replaces all characters not matching
@@ -2134,8 +2135,10 @@ def tagify(s, maxlen=None):
         >>> tagify("HTTP Web Title", maxlen=8)
         'http-web'
     """
+    if delimiter is None:
+        delimiter = "-"
     ret = str(s).lower()
-    return tag_filter_regex.sub("-", ret)[:maxlen].strip("-")
+    return tag_filter_regex.sub(delimiter, ret)[:maxlen].strip(delimiter)
 
 
 def memory_status():
diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 263dc9dce..37530e57f 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -74,10 +74,10 @@ async def request(self, *args, **kwargs):
         async with self._acatch(url, raise_error):
             if self.http_debug:
                 logstr = f"Web request: {str(args)}, {str(kwargs)}"
-                self.log.trace(logstr)
+                log.trace(logstr)
             response = await client.request(*args, **kwargs)
             if self.http_debug:
-                self.log.trace(
+                log.trace(
                     f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}"
                 )
             return response
@@ -106,10 +106,27 @@ def new_task(url):
                     new_task(urls.pop(0))
 
     async def download(self, url, **kwargs):
-        follow_redirects = kwargs.pop("follow_redirects", True)
+        warn = kwargs.pop("warn", True)
         filename = kwargs.pop("filename")
+        raise_error = kwargs.get("raise_error", False)
+        try:
+            content, response = await self.stream_request(url, **kwargs)
+            log.debug(f"Download result: HTTP {response.status_code}")
+            response.raise_for_status()
+            with open(filename, "wb") as f:
+                f.write(content)
+            return filename
+        except httpx.HTTPError as e:
+            log_fn = log.verbose
+            if warn:
+                log_fn = log.warning
+            log_fn(f"Failed to download {url}: {e}")
+            if raise_error:
+                raise
+
+    async def stream_request(self, url, **kwargs):
+        follow_redirects = kwargs.pop("follow_redirects", True)
         max_size = kwargs.pop("max_size", None)
-        warn = kwargs.pop("warn", True)
         raise_error = kwargs.pop("raise_error", False)
         if max_size is not None:
             max_size = human_to_bytes(max_size)
@@ -119,29 +136,23 @@ async def download(self, url, **kwargs):
         try:
             total_size = 0
             chunk_size = 8192
+            chunks = []
 
             async with self._acatch(url, raise_error=True), self.web_client.stream(url=url, **kwargs) as response:
-                status_code = getattr(response, "status_code", 0)
-                self.log.debug(f"Download result: HTTP {status_code}")
-                if status_code != 0:
-                    response.raise_for_status()
-                    with open(filename, "wb") as f:
-                        agen = response.aiter_bytes(chunk_size=chunk_size)
-                        async for chunk in agen:
-                            if max_size is not None and total_size + chunk_size > max_size:
-                                self.log.verbose(
-                                    f"Filesize of {url} exceeds {bytes_to_human(max_size)}, file will be truncated"
-                                )
-                                agen.aclose()
-                                break
-                            total_size += chunk_size
-                            f.write(chunk)
-                    return True
+                agen = response.aiter_bytes(chunk_size=chunk_size)
+                async for chunk in agen:
+                    _chunk_size = len(chunk)
+                    if max_size is not None and total_size + _chunk_size > max_size:
+                        log.verbose(
+                            f"Size of response from {url} exceeds {bytes_to_human(max_size)}, file will be truncated"
+                        )
+                        agen.aclose()
+                        break
+                    total_size += _chunk_size
+                    chunks.append(chunk)
+                return b"".join(chunks), response
         except httpx.HTTPError as e:
-            log_fn = self.log.verbose
-            if warn:
-                log_fn = self.log.warning
-            log_fn(f"Failed to download {url}: {e}")
+            self.debug(f"Error requesting {url}: {e}")
             if raise_error:
                 raise
 
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 819b5d6bf..521da510e 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -6,7 +6,7 @@
 from bs4 import BeautifulSoup
 
 from bbot.core.engine import EngineClient
-from bbot.errors import WordlistError, CurlError
+from bbot.errors import WordlistError, CurlError, WebError
 
 from bs4 import MarkupResemblesLocatorWarning
 from bs4.builder import XMLParsedAsHTMLWarning
@@ -22,6 +22,7 @@
 class WebHelper(EngineClient):
 
     SERVER_CLASS = HTTPEngine
+    ERROR_CLASS = WebError
 
     """
     Main utility class for managing HTTP operations in BBOT. It serves as a wrapper around the BBOTAsyncClient,
@@ -552,3 +553,58 @@ def is_login_page(self, html):
             if user_fields and pass_fields:
                 return True
         return False
+
+    def response_to_json(self, response):
+        """
+        Convert web response to JSON object, similar to the output of `httpx -irr -json`
+        """
+
+        if response is None:
+            return
+
+        import mmh3
+        from datetime import datetime
+        from hashlib import md5, sha256
+        from bbot.core.helpers.misc import tagify, urlparse, split_host_port, smart_decode
+
+        request = response.request
+        url = str(request.url)
+        parsed_url = urlparse(url)
+        netloc = parsed_url.netloc
+        scheme = parsed_url.scheme.lower()
+        host, port = split_host_port(f"{scheme}://{netloc}")
+
+        raw_headers = "\r\n".join([f"{k}: {v}" for k, v in response.headers.items()])
+        raw_headers_encoded = raw_headers.encode()
+
+        headers = {}
+        for k, v in response.headers.items():
+            k = tagify(k, delimiter="_")
+            headers[k] = v
+
+        j = {
+            "timestamp": datetime.now().isoformat(),
+            "hash": {
+                "body_md5": md5(response.content).hexdigest(),
+                "body_mmh3": mmh3.hash(response.content),
+                "body_sha256": sha256(response.content).hexdigest(),
+                # "body_simhash": "TODO",
+                "header_md5": md5(raw_headers_encoded).hexdigest(),
+                "header_mmh3": mmh3.hash(raw_headers_encoded),
+                "header_sha256": sha256(raw_headers_encoded).hexdigest(),
+                # "header_simhash": "TODO",
+            },
+            "header": headers,
+            "body": smart_decode(response.content),
+            "content_type": headers.get("content_type", "").split(";")[0].strip(),
+            "url": url,
+            "host": str(host),
+            "port": port,
+            "scheme": scheme,
+            "method": response.request.method,
+            "path": parsed_url.path,
+            "raw_header": raw_headers,
+            "status_code": response.status_code,
+        }
+
+        return j
diff --git a/bbot/errors.py b/bbot/errors.py
index 153601813..30948835d 100644
--- a/bbot/errors.py
+++ b/bbot/errors.py
@@ -76,3 +76,11 @@ class PresetAbortError(PresetConditionError):
 
 class BBOTEngineError(BBOTError):
     pass
+
+
+class WebError(BBOTEngineError):
+    pass
+
+
+class DNSError(BBOTEngineError):
+    pass
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 18243c393..44d4e6a50 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -70,6 +70,7 @@ def parse_results(self, r, query=None):
                 yield hostname
 
     async def query(self, query, parse_fn=None, request_fn=None):
+        self.hugesuccess(query)
         if parse_fn is None:
             parse_fn = self.parse_results
         if request_fn is None:
@@ -86,7 +87,7 @@ async def query(self, query, parse_fn=None, request_fn=None):
                     self.info(
                         f'Error parsing results for query "{query}" (status code {response.status_code})', trace=True
                     )
-                    self.log.trace(response.text)
+                    self.log.trace(repr(response.text))
                 else:
                     self.info(f'Error parsing results for "{query}": {e}', trace=True)
                 return
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index c9f3b49da..5d104acb1 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -178,7 +178,11 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.subdomain_depth("a.evilcorp.com") == 1
     assert helpers.subdomain_depth("a.s.d.f.evilcorp.notreal") == 4
 
+    assert helpers.split_host_port("http://evilcorp.co.uk") == ("evilcorp.co.uk", 80)
     assert helpers.split_host_port("https://evilcorp.co.uk") == ("evilcorp.co.uk", 443)
+    assert helpers.split_host_port("ws://evilcorp.co.uk") == ("evilcorp.co.uk", 80)
+    assert helpers.split_host_port("wss://evilcorp.co.uk") == ("evilcorp.co.uk", 443)
+    assert helpers.split_host_port("WSS://evilcorp.co.uk") == ("evilcorp.co.uk", 443)
     assert helpers.split_host_port("http://evilcorp.co.uk:666") == ("evilcorp.co.uk", 666)
     assert helpers.split_host_port("evilcorp.co.uk:666") == ("evilcorp.co.uk", 666)
     assert helpers.split_host_port("evilcorp.co.uk") == ("evilcorp.co.uk", None)
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index 21fefe5d6..e73204789 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -1,5 +1,4 @@
 import re
-import httpx
 
 from ..bbot_fixtures import *
 
@@ -41,12 +40,29 @@ async def test_web_engine(bbot_scanner, bbot_httpserver, httpx_mock):
     assert file_content == web_body
 
     # raise_error=True
-    with pytest.raises(httpx.TimeoutException):
+    with pytest.raises(WebError):
         await scan.helpers.request("http://www.example.com/", raise_error=True)
 
 
 @pytest.mark.asyncio
-async def test_web_helpers(bbot_scanner, bbot_httpserver):
+async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock):
+
+    # json conversion
+    scan = bbot_scanner("evilcorp.com")
+    url = "http://www.evilcorp.com/json_test?a=b"
+    httpx_mock.add_response(url=url, text="hello\nworld")
+    response = await scan.helpers.web.request(url)
+    j = scan.helpers.response_to_json(response)
+    assert j["status_code"] == 200
+    assert j["host"] == "www.evilcorp.com"
+    assert j["scheme"] == "http"
+    assert j["method"] == "GET"
+    assert j["port"] == 80
+    assert j["path"] == "/json_test"
+    assert j["body"] == "hello\nworld"
+    assert j["content_type"] == "text/plain"
+    assert j["url"] == "http://www.evilcorp.com/json_test?a=b"
+
     scan1 = bbot_scanner("8.8.8.8")
     scan2 = bbot_scanner("127.0.0.1")
 
@@ -118,6 +134,8 @@ async def test_web_helpers(bbot_scanner, bbot_httpserver):
     filename = await scan1.helpers.download(url)
     assert filename is None
     assert not scan1.helpers.is_cached(url)
+    with pytest.raises(WebError):
+        filename = await scan1.helpers.download(url, raise_error=True)
 
     # wordlist
     path = "/test_http_helpers_wordlist"

From f98d305a79cb68ed6a733a09c0b9b0fb21651989 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 14:33:46 -0400
Subject: [PATCH 124/220] fix engine test

---
 bbot/test/test_step_1/test_engine.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_engine.py b/bbot/test/test_step_1/test_engine.py
index f24ea59c0..a8a4156d1 100644
--- a/bbot/test/test_step_1/test_engine.py
+++ b/bbot/test/test_step_1/test_engine.py
@@ -96,7 +96,7 @@ async def yield_stuff(self, n):
     yield_cancelled = False
     yield_errored = False
     agen = test_engine.yield_stuff(None)
-    with pytest.raises(TypeError):
+    with pytest.raises(BBOTEngineError):
         async for _ in agen:
             pass
     assert yield_cancelled == False
@@ -138,7 +138,7 @@ async def yield_stuff(self, n):
     return_finished = False
     return_cancelled = False
     return_errored = False
-    with pytest.raises(TypeError):
+    with pytest.raises(BBOTEngineError):
         result = await test_engine.return_thing(None)
     assert return_started == True
     assert return_finished == False

From b16f7d9432c2394422265149f39388cc8e8b8b6a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 15:15:37 -0400
Subject: [PATCH 125/220] better error handling for filter_event() and
 _incoming_dup_hash()

---
 bbot/modules/base.py                     | 14 ++++++++++++--
 bbot/modules/github_codesearch.py        |  3 ++-
 bbot/modules/templates/github.py         |  4 ++--
 bbot/modules/templates/subdomain_enum.py |  6 +++++-
 4 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index d054955a3..65de3299e 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -738,7 +738,12 @@ async def _event_postcheck_inner(self, event):
 
         # custom filtering
         async with self.scan._acatch(context=self.filter_event):
-            filter_result = await self.filter_event(event)
+            try:
+                filter_result = await self.filter_event(event)
+            except Exception as e:
+                msg = f"Unhandled exception in {self.name}.filter_event({event}): {e}"
+                self.error(msg)
+                return False, msg
             msg = str(self._custom_filter_criteria_msg)
             with suppress(ValueError, TypeError):
                 filter_result, reason = filter_result
@@ -886,7 +891,12 @@ def is_incoming_duplicate(self, event, add=False):
         if event.type in ("FINISHED",):
             return False, ""
         reason = ""
-        event_hash = self._incoming_dedup_hash(event)
+        try:
+            event_hash = self._incoming_dedup_hash(event)
+        except Exception as e:
+            msg = f"Unhandled exception in {self.name}._incoming_dedup_hash({event}): {e}"
+            self.error(msg)
+            return True, msg
         with suppress(TypeError, ValueError):
             event_hash, reason = event_hash
         is_dup = event_hash in self._incoming_dup_tracker
diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py
index 634b38f58..756645c13 100644
--- a/bbot/modules/github_codesearch.py
+++ b/bbot/modules/github_codesearch.py
@@ -1,7 +1,8 @@
 from bbot.modules.templates.github import github
+from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
-class github_codesearch(github):
+class github_codesearch(github, subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["CODE_REPOSITORY", "URL_UNVERIFIED"]
     flags = ["passive", "subdomain-enum", "safe", "code-enum"]
diff --git a/bbot/modules/templates/github.py b/bbot/modules/templates/github.py
index fcfeb0934..4169647ce 100644
--- a/bbot/modules/templates/github.py
+++ b/bbot/modules/templates/github.py
@@ -1,7 +1,7 @@
-from bbot.modules.templates.subdomain_enum import subdomain_enum
+from bbot.modules.base import BaseModule
 
 
-class github(subdomain_enum):
+class github(BaseModule):
     """
     A template module for use of the GitHub API
     Inherited by several other github modules.
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 65c0cd6d3..3dde7a148 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -67,7 +67,11 @@ def make_query(self, event):
             if self.scan.in_scope(p):
                 query = p
                 break
-        return ".".join([s for s in query.split(".") if s != "_wildcard"])
+        try:
+            return ".".join([s for s in query.split(".") if s != "_wildcard"])
+        except Exception:
+            self.critical(query)
+            raise
 
     def parse_results(self, r, query=None):
         json = r.json()

From e95d3a6c80a76b36fd159fb1341d66f55f3e0bd5 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 16:25:11 -0400
Subject: [PATCH 126/220] use request_batch in iis_shortnames

---
 bbot/core/helpers/web/engine.py    |  40 +++++++--
 bbot/core/helpers/web/web.py       |   4 +-
 bbot/modules/iis_shortnames.py     | 133 +++++++++++++++--------------
 bbot/modules/paramminer_headers.py |   8 +-
 bbot/test/test_step_1/test_web.py  |  49 +++++++----
 5 files changed, 141 insertions(+), 93 deletions(-)

diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py
index 37530e57f..2bbacbc71 100644
--- a/bbot/core/helpers/web/engine.py
+++ b/bbot/core/helpers/web/engine.py
@@ -85,14 +85,16 @@ async def request(self, *args, **kwargs):
     async def request_batch(self, urls, *args, threads=10, **kwargs):
         tasks = {}
 
-        def new_task(url):
-            task = asyncio.create_task(self.request(url, *args, **kwargs))
-            tasks[task] = url
-
         urls = list(urls)
+
+        def new_task():
+            if urls:
+                url = urls.pop(0)
+                task = asyncio.create_task(self.request(url, *args, **kwargs))
+                tasks[task] = url
+
         for _ in range(threads):  # Start initial batch of tasks
-            if urls:  # Ensure there are args to process
-                new_task(urls.pop(0))
+            new_task()
 
         while tasks:  # While there are tasks pending
             # Wait for the first task to complete
@@ -102,8 +104,30 @@ def new_task(url):
                 response = task.result()
                 url = tasks.pop(task)
                 yield (url, response)
-                if urls:  # Start a new task for each one completed, if URLs remain
-                    new_task(urls.pop(0))
+                new_task()
+
+    async def request_custom_batch(self, urls_and_kwargs, threads=10):
+        tasks = {}
+        urls_and_kwargs = list(urls_and_kwargs)
+
+        def new_task():
+            if urls_and_kwargs:  # Ensure there are args to process
+                url, kwargs, custom_tracker = urls_and_kwargs.pop(0)
+                task = asyncio.create_task(self.request(url, **kwargs))
+                tasks[task] = (url, kwargs, custom_tracker)
+
+        for _ in range(threads):  # Start initial batch of tasks
+            new_task()
+
+        while tasks:  # While there are tasks pending
+            # Wait for the first task to complete
+            done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
+
+            for task in done:
+                response = task.result()
+                url, kwargs, custom_tracker = tasks.pop(task)
+                yield (url, kwargs, custom_tracker, response)
+                new_task()
 
     async def download(self, url, **kwargs):
         warn = kwargs.pop("warn", True)
diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py
index 521da510e..d6a02241a 100644
--- a/bbot/core/helpers/web/web.py
+++ b/bbot/core/helpers/web/web.py
@@ -123,7 +123,7 @@ async def request_batch(self, urls, *args, **kwargs):
         async for _ in self.run_and_yield("request_batch", urls, *args, **kwargs):
             yield _
 
-    async def request_custom_batch(self, urls_and_args):
+    async def request_custom_batch(self, urls_and_kwargs):
         """
         Make web requests in parallel with custom options for each request. Yield responses as they come in.
 
@@ -145,7 +145,7 @@ async def request_custom_batch(self, urls_and_args):
             >>>     if response is not None and response.status_code == 200:
             >>>         self.hugesuccess(response)
         """
-        async for _ in self.run_and_yield("request_custom_batch", urls_and_args):
+        async for _ in self.run_and_yield("request_custom_batch", urls_and_kwargs):
             yield _
 
     async def download(self, url, **kwargs):
diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index 94d325df8..047d247ea 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -34,10 +34,22 @@ async def detect(self, target):
         control_url = f"{target}{random_string}*~1*/a.aspx"
         test_url = f"{target}*~1*/a.aspx"
 
+        urls_and_kwargs = []
         for method in ["GET", "POST", "OPTIONS", "DEBUG", "HEAD", "TRACE"]:
-            control = await self.helpers.request(method=method, url=control_url, allow_redirects=False, timeout=10)
-            test = await self.helpers.request(method=method, url=test_url, allow_redirects=False, timeout=10)
-            if (control != None) and (test != None):
+            kwargs = dict(method=method, allow_redirects=False, timeout=10)
+            urls_and_kwargs.append((control_url, kwargs, method))
+            urls_and_kwargs.append((test_url, kwargs, method))
+
+        results = {}
+        async for url, kwargs, method, response in self.helpers.request_custom_batch(urls_and_kwargs):
+            try:
+                results[method][url] = response
+            except KeyError:
+                results[method] = {url: response}
+        for method, result in results.items():
+            control = results[method].get(control_url, None)
+            test = results[method].get(test_url, None)
+            if (result != None) and (test != None):
                 if control.status_code != test.status_code:
                     technique = f"{str(control.status_code)}/{str(test.status_code)} HTTP Code"
                     detections.append((method, test.status_code, technique))
@@ -109,32 +121,23 @@ async def threaded_request(self, method, url, affirmative_status_code, c):
     async def solve_valid_chars(self, method, target, affirmative_status_code):
         confirmed_chars = []
         confirmed_exts = []
-        tasks = []
         suffix = "/a.aspx"
 
+        urls_and_kwargs = []
+        kwargs = dict(method=method, allow_redirects=False, retries=2, timeout=10)
         for c in valid_chars:
-            payload = encode_all(f"*{c}*~1*")
-            url = f"{target}{payload}{suffix}"
-            task = self.threaded_request(method, url, affirmative_status_code, c)
-            tasks.append(task)
-
-        async for task in self.helpers.as_completed(tasks):
-            result, c = await task
-            if result:
-                confirmed_chars.append(c)
-
-        tasks = []
-
-        for c in valid_chars:
-            payload = encode_all(f"*~1*{c}*")
-            url = f"{target}{payload}{suffix}"
-            task = self.threaded_request(method, url, affirmative_status_code, c)
-            tasks.append(task)
+            for file_part in ("stem", "ext"):
+                payload = encode_all(f"*{c}*~1*")
+                url = f"{target}{payload}{suffix}"
+                urls_and_kwargs.append((url, kwargs, (c, file_part)))
 
-        async for task in self.helpers.as_completed(tasks):
-            result, c = await task
-            if result:
-                confirmed_exts.append(c)
+        async for url, kwargs, (c, file_part), response in self.helpers.request_custom_batch(urls_and_kwargs):
+            if response is not None:
+                if response.status_code == affirmative_status_code:
+                    if file_part == "stem":
+                        confirmed_chars.append(c)
+                    elif file_part == "ext":
+                        confirmed_exts.append(c)
 
         return confirmed_chars, confirmed_exts
 
@@ -153,53 +156,55 @@ async def solve_shortname_recursive(
         url_hint_list = []
         found_results = False
 
-        tasks = []
-
         cl = ext_char_list if extension_mode == True else char_list
 
+        urls_and_kwargs = []
+
         for c in cl:
             suffix = "/a.aspx"
             wildcard = "*" if extension_mode else "*~1*"
             payload = encode_all(f"{prefix}{c}{wildcard}")
             url = f"{target}{payload}{suffix}"
-            task = self.threaded_request(method, url, affirmative_status_code, c)
-            tasks.append(task)
-
-        async for task in self.helpers.as_completed(tasks):
-            result, c = await task
-            if result:
-                found_results = True
-                node_count += 1
-                safety_counter.counter += 1
-                if safety_counter.counter > 3000:
-                    raise IISShortnamesError(f"Exceeded safety counter threshold ({safety_counter.counter})")
-                self.verbose(f"node_count: {str(node_count)} for node: {target}")
-                if node_count > self.config.get("max_node_count"):
-                    self.warning(
-                        f"iis_shortnames: max_node_count ({str(self.config.get('max_node_count'))}) exceeded for node: {target}. Affected branch will be terminated."
+            kwargs = dict(method=method)
+            urls_and_kwargs.append((url, kwargs, c))
+
+        async for url, kwargs, c, response in self.helpers.request_custom_batch(urls_and_kwargs):
+            if response is not None:
+                if response.status_code == affirmative_status_code:
+                    found_results = True
+                    node_count += 1
+                    safety_counter.counter += 1
+                    if safety_counter.counter > 3000:
+                        raise IISShortnamesError(f"Exceeded safety counter threshold ({safety_counter.counter})")
+                    self.verbose(f"node_count: {str(node_count)} for node: {target}")
+                    if node_count > self.config.get("max_node_count"):
+                        self.warning(
+                            f"iis_shortnames: max_node_count ({str(self.config.get('max_node_count'))}) exceeded for node: {target}. Affected branch will be terminated."
+                        )
+                        return url_hint_list
+
+                    # check to make sure the file isn't shorter than 6 characters
+                    wildcard = "~1*"
+                    payload = encode_all(f"{prefix}{c}{wildcard}")
+                    url = f"{target}{payload}{suffix}"
+                    r = await self.helpers.request(
+                        method=method, url=url, allow_redirects=False, retries=2, timeout=10
+                    )
+                    if r is not None:
+                        if r.status_code == affirmative_status_code:
+                            url_hint_list.append(f"{prefix}{c}")
+
+                    url_hint_list += await self.solve_shortname_recursive(
+                        safety_counter,
+                        method,
+                        target,
+                        f"{prefix}{c}",
+                        affirmative_status_code,
+                        char_list,
+                        ext_char_list,
+                        extension_mode,
+                        node_count=node_count,
                     )
-                    return url_hint_list
-
-                # check to make sure the file isn't shorter than 6 characters
-                wildcard = "~1*"
-                payload = encode_all(f"{prefix}{c}{wildcard}")
-                url = f"{target}{payload}{suffix}"
-                r = await self.helpers.request(method=method, url=url, allow_redirects=False, retries=2, timeout=10)
-                if r is not None:
-                    if r.status_code == affirmative_status_code:
-                        url_hint_list.append(f"{prefix}{c}")
-
-                url_hint_list += await self.solve_shortname_recursive(
-                    safety_counter,
-                    method,
-                    target,
-                    f"{prefix}{c}",
-                    affirmative_status_code,
-                    char_list,
-                    ext_char_list,
-                    extension_mode,
-                    node_count=node_count,
-                )
         if len(prefix) > 0 and found_results == False:
             url_hint_list.append(f"{prefix}")
             self.verbose(f"Found new (possibly partial) URL_HINT: {prefix} from node {target}")
diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index 28c94b9b6..d1ee67b6b 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -179,9 +179,9 @@ async def count_test(self, url):
             return
         if str(baseline.status_code)[0] in ("4", "5"):
             return
-        for count, args, kwargs in self.gen_count_args(url):
-            r = await self.helpers.request(*args, **kwargs)
-            if r is not None and not ((str(r.status_code)[0] in ("4", "5"))):
+        urls_and_args = list(self.gen_count_args(url))
+        async for _url, kwargs, count, response in self.helpers.request_custom_batch(urls_and_args):
+            if response is not None and not ((str(response.status_code)[0] in ("4", "5"))):
                 return count
 
     def gen_count_args(self, url):
@@ -192,7 +192,7 @@ def gen_count_args(self, url):
             fake_headers = {}
             for i in range(0, header_count):
                 fake_headers[self.rand_string(14)] = self.rand_string(14)
-            yield header_count, (url,), {"headers": fake_headers}
+            yield url, {"headers": fake_headers}, header_count
             header_count -= 5
 
     async def load_extracted_words(self, body, content_type):
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index e73204789..e67efa134 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -6,38 +6,57 @@
 @pytest.mark.asyncio
 async def test_web_engine(bbot_scanner, bbot_httpserver, httpx_mock):
 
-    web_body = "hello_there" * 1000
-    url = bbot_httpserver.url_for("/test")
-    bbot_httpserver.expect_request(uri="/test").respond_with_data(web_body)
+    from werkzeug.wrappers import Response
+
+    def server_handler(request):
+        return Response(f"{request.url}: {request.headers}")
+
+    base_url = bbot_httpserver.url_for("/test/")
+    bbot_httpserver.expect_request(uri=re.compile(r"/test/\d+")).respond_with_handler(server_handler)
 
     scan = bbot_scanner()
 
     # request
-    response = await scan.helpers.request(url)
-    assert response.status_code > 0
-    assert response.text == web_body
+    response = await scan.helpers.request(f"{base_url}1")
+    assert response.status_code == 200
+    assert response.text.startswith(f"{base_url}1: ")
+
+    num_urls = 100
 
     # request_batch
-    responses = [r async for r in scan.helpers.request_batch([url] * 100)]
+    urls = [f"{base_url}{i}" for i in range(num_urls)]
+    responses = [r async for r in scan.helpers.request_batch(urls)]
     assert len(responses) == 100
-    assert all([r[0] == url for r in responses])
-    assert all([r[1].status_code > 0 and r[1].text == web_body for r in responses])
+    assert all([r[1].status_code == 200 and r[1].text.startswith(f"{r[0]}: ") for r in responses])
 
     # request_batch w/ cancellation
-    agen = scan.helpers.request_batch([url] * 100)
+    agen = scan.helpers.request_batch(urls)
     async for url, response in agen:
-        assert response.text == web_body
+        assert response.text.startswith(base_url)
         await agen.aclose()
         break
 
-    import asyncio
-
-    await asyncio.sleep(5)
+    # request_custom_batch
+    urls_and_kwargs = [(urls[i], {"headers": {f"h{i}": f"v{i}"}}, i) for i in range(num_urls)]
+    results = [r async for r in scan.helpers.request_custom_batch(urls_and_kwargs)]
+    assert len(responses) == 100
+    for i in range(num_urls):
+        url, kwargs, custom_tracker, response = results[i]
+        expected_url = f"{base_url}{i}"
+        assert url == expected_url
+        assert "headers" in kwargs
+        assert f"h{i}" in kwargs["headers"]
+        assert kwargs["headers"][f"h{i}"] == f"v{i}"
+        assert custom_tracker == i
+        assert response.status_code == 200
+        assert response.text.startswith(f"{expected_url}: ")
+        assert f"H{i}: v{i}" in response.text
 
     # download
+    url = f"{base_url}999"
     filename = await scan.helpers.download(url)
     file_content = open(filename).read()
-    assert file_content == web_body
+    assert file_content.startswith(f"{url}: ")
 
     # raise_error=True
     with pytest.raises(WebError):

From fdb89511f925cc4b0a7e17c9b7706af3c8d9fb25 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 16:40:57 -0400
Subject: [PATCH 127/220] fix wonky test

---
 bbot/test/test_step_1/test_web.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index e67efa134..f762efc45 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -42,14 +42,12 @@ def server_handler(request):
     assert len(responses) == 100
     for i in range(num_urls):
         url, kwargs, custom_tracker, response = results[i]
-        expected_url = f"{base_url}{i}"
-        assert url == expected_url
         assert "headers" in kwargs
         assert f"h{i}" in kwargs["headers"]
         assert kwargs["headers"][f"h{i}"] == f"v{i}"
         assert custom_tracker == i
         assert response.status_code == 200
-        assert response.text.startswith(f"{expected_url}: ")
+        assert response.text.startswith(f"{url}: ")
         assert f"H{i}: v{i}" in response.text
 
     # download

From 9d881f233babe8f984bb8e5db13776bfa39b1db9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 17:34:01 -0400
Subject: [PATCH 128/220] fix tests

---
 bbot/test/test_step_1/test_web.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index f762efc45..d92f239d7 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -40,15 +40,14 @@ def server_handler(request):
     urls_and_kwargs = [(urls[i], {"headers": {f"h{i}": f"v{i}"}}, i) for i in range(num_urls)]
     results = [r async for r in scan.helpers.request_custom_batch(urls_and_kwargs)]
     assert len(responses) == 100
-    for i in range(num_urls):
-        url, kwargs, custom_tracker, response = results[i]
+    for result in results:
+        url, kwargs, custom_tracker, response = result
         assert "headers" in kwargs
-        assert f"h{i}" in kwargs["headers"]
-        assert kwargs["headers"][f"h{i}"] == f"v{i}"
-        assert custom_tracker == i
+        assert f"h{custom_tracker}" in kwargs["headers"]
+        assert kwargs["headers"][f"h{custom_tracker}"] == f"v{custom_tracker}"
         assert response.status_code == 200
         assert response.text.startswith(f"{url}: ")
-        assert f"H{i}: v{i}" in response.text
+        assert f"H{custom_tracker}: v{custom_tracker}" in response.text
 
     # download
     url = f"{base_url}999"

From 2fc69414c2d73319116f90c2c74b93d13558ac73 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 22:31:47 -0400
Subject: [PATCH 129/220] fix paramminer tests

---
 bbot/modules/paramminer_headers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index d1ee67b6b..28c94b9b6 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -179,9 +179,9 @@ async def count_test(self, url):
             return
         if str(baseline.status_code)[0] in ("4", "5"):
             return
-        urls_and_args = list(self.gen_count_args(url))
-        async for _url, kwargs, count, response in self.helpers.request_custom_batch(urls_and_args):
-            if response is not None and not ((str(response.status_code)[0] in ("4", "5"))):
+        for count, args, kwargs in self.gen_count_args(url):
+            r = await self.helpers.request(*args, **kwargs)
+            if r is not None and not ((str(r.status_code)[0] in ("4", "5"))):
                 return count
 
     def gen_count_args(self, url):
@@ -192,7 +192,7 @@ def gen_count_args(self, url):
             fake_headers = {}
             for i in range(0, header_count):
                 fake_headers[self.rand_string(14)] = self.rand_string(14)
-            yield url, {"headers": fake_headers}, header_count
+            yield header_count, (url,), {"headers": fake_headers}
             header_count -= 5
 
     async def load_extracted_words(self, body, content_type):

From a6ef4b3d651300416cb7a12dfab65d226c84b2c4 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 13 May 2024 22:36:08 -0400
Subject: [PATCH 130/220] fix ctrl+c spewing

---
 bbot/core/engine.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index de09cc93b..38e9cd30c 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -209,6 +209,8 @@ async def run_and_return(self, client_id, command_fn, *args, **kwargs):
             self.log.debug(f"{self.name} run-and-return {command_fn.__name__}({args}, {kwargs})")
             try:
                 result = await command_fn(*args, **kwargs)
+            except (asyncio.CancelledError, KeyboardInterrupt):
+                pass
             except BaseException as e:
                 error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
                 trace = traceback.format_exc()
@@ -231,6 +233,8 @@ async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
                 async for _ in command_fn(*args, **kwargs):
                     await self.send_socket_multipart(client_id, _)
                 await self.send_socket_multipart(client_id, {"_s": None})
+            except (asyncio.CancelledError, KeyboardInterrupt):
+                pass
             except BaseException as e:
                 error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
                 trace = traceback.format_exc()

From e9b13fd7efd6ef251ee7fad79d59275f99485eae Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 14 May 2024 10:06:41 -0400
Subject: [PATCH 131/220] update httpx error handling

---
 bbot/modules/output/http.py   | 5 ++---
 bbot/modules/output/splunk.py | 5 ++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py
index 18182056a..7bb1d301e 100644
--- a/bbot/modules/output/http.py
+++ b/bbot/modules/output/http.py
@@ -1,5 +1,4 @@
-from httpx import RequestError
-
+from bbot.errors import WebError
 from bbot.modules.output.base import BaseOutputModule
 
 
@@ -59,6 +58,6 @@ async def handle_event(self, event):
                     raise_error=True,
                 )
                 break
-            except RequestError as e:
+            except WebError as e:
                 self.warning(f"Error sending {event}: {e}, retrying...")
                 await self.helpers.sleep(1)
diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py
index 00d70876b..2c5b08cfc 100644
--- a/bbot/modules/output/splunk.py
+++ b/bbot/modules/output/splunk.py
@@ -1,5 +1,4 @@
-from httpx import RequestError
-
+from bbot.errors import WebError
 from bbot.modules.output.base import BaseOutputModule
 
 
@@ -54,6 +53,6 @@ async def handle_event(self, event):
                     raise_error=True,
                 )
                 break
-            except RequestError as e:
+            except WebError as e:
                 self.warning(f"Error sending {event}: {e}, retrying...")
                 await self.helpers.sleep(1)

From bfd62debc96ca387eaf7da6c6df67330e813b036 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 14 May 2024 14:41:33 -0400
Subject: [PATCH 132/220] working on generic_ssrf tests

---
 bbot/test/conftest.py                                     | 8 +++++++-
 .../test_step_2/module_tests/test_module_generic_ssrf.py  | 2 ++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 9d0a8fab7..9d92a9315 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -5,6 +5,7 @@
 import asyncio
 import logging
 from pathlib import Path
+from contextlib import suppress
 from omegaconf import OmegaConf
 from pytest_httpserver import HTTPServer
 
@@ -119,17 +120,22 @@ def __init__(self):
         self.interactions = []
         self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef"
         self.stop = False
+        self.poll_task = None
 
     def mock_interaction(self, subdomain_tag):
         self.interactions.append(subdomain_tag)
 
     async def register(self, callback=None):
         if callable(callback):
-            asyncio.create_task(self.poll_loop(callback))
+            self.poll_task = asyncio.create_task(self.poll_loop(callback))
         return "fakedomain.fakeinteractsh.com"
 
     async def deregister(self, callback=None):
         self.stop = True
+        if self.poll_task is not None:
+            self.poll_task.cancel()
+            with suppress(BaseException):
+                await self.poll_task
 
     async def poll_loop(self, callback=None):
         while not self.stop:
diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
index 370dd151a..f5f9976ba 100644
--- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
+++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
@@ -38,6 +38,8 @@ async def setup_after_prep(self, module_test):
         module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)
 
     def check(self, module_test, events):
+        for e in events:
+            self.log.critical(e)
         assert any(
             e.type == "VULNERABILITY" and "Out-of-band interaction: [Generic SSRF (GET)]" in e.data["description"]
             for e in events

From 9c2c5416b56b178f8a4f40b15d757e7f2a67d061 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 14 May 2024 16:45:50 -0400
Subject: [PATCH 133/220] portscanner rework

---
 bbot/core/event/base.py                       |   2 +-
 bbot/core/modules.py                          |   6 +-
 bbot/core/shared_deps.py                      |  30 ++
 bbot/modules/internal/speculate.py            |   2 +-
 bbot/modules/masscan.py                       | 278 ------------------
 bbot/modules/nmap.py                          | 138 ---------
 bbot/scanner/preset/args.py                   |   2 +-
 bbot/scanner/preset/preset.py                 |  16 +-
 bbot/scanner/scanner.py                       |   6 +-
 .../test_module_asset_inventory.py            |   4 +-
 .../module_tests/test_module_masscan.py       |  40 ---
 .../module_tests/test_module_nmap.py          |  79 -----
 .../module_tests/test_module_speculate.py     |   2 +-
 docs/dev/helpers/command.md                   |   2 +-
 docs/how_it_works.md                          |  16 +-
 docs/modules/nuclei.md                        |   8 +-
 docs/scanning/advanced.md                     |   4 +-
 docs/scanning/configuration.md                |   2 +-
 18 files changed, 66 insertions(+), 571 deletions(-)
 delete mode 100644 bbot/modules/masscan.py
 delete mode 100644 bbot/modules/nmap.py
 delete mode 100644 bbot/test/test_step_2/module_tests/test_module_masscan.py
 delete mode 100644 bbot/test/test_step_2/module_tests/test_module_nmap.py

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 66116d87d..908fec9fa 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1269,7 +1269,7 @@ def make_event(
     Examples:
         If inside a module, e.g. from within its `handle_event()`:
         >>> self.make_event("1.2.3.4", source=event)
-        IP_ADDRESS("1.2.3.4", module=nmap, tags={'ipv4', 'distance-1'})
+        IP_ADDRESS("1.2.3.4", module=portscan, tags={'ipv4', 'distance-1'})
 
         If you're outside a module but you have a scan object:
         >>> scan.make_event("1.2.3.4", source=scan.root_event)
diff --git a/bbot/core/modules.py b/bbot/core/modules.py
index b9ae83af5..c5eb8f902 100644
--- a/bbot/core/modules.py
+++ b/bbot/core/modules.py
@@ -487,7 +487,7 @@ def recommend_dependencies(self, modules):
         """
         resolve_choices = {}
         # step 1: build a dictionary containing event types and their associated modules
-        # {"IP_ADDRESS": set("nmap", "ipneighbor", ...)}
+        # {"IP_ADDRESS": set("masscan", "ipneighbor", ...)}
         watched = {}
         produced = {}
         for modname in modules:
@@ -560,11 +560,11 @@ def modules_table(self, modules=None, mod_type=None):
             str: A formatted table string.
 
         Examples:
-            >>> print(modules_table(["nmap"]))
+            >>> print(modules_table(["portscan"]))
             +----------+--------+-----------------+------------------------------+-------------------------------+----------------------+-------------------+
             | Module   | Type   | Needs API Key   | Description                  | Flags                         | Consumed Events      | Produced Events   |
             +==========+========+=================+==============================+===============================+======================+===================+
-            | nmap     | scan   | No              | Execute port scans with nmap | active, aggressive, portscan, | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT     |
+            | portscan | scan   | No              | Execute port scans           | active, aggressive, portscan, | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT     |
             |          |        |                 |                              | web-thorough                  |                      |                   |
             +----------+--------+-----------------+------------------------------+-------------------------------+----------------------+-------------------+
         """
diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py
index 751117752..ebc3759e4 100644
--- a/bbot/core/shared_deps.py
+++ b/bbot/core/shared_deps.py
@@ -111,6 +111,36 @@
     },
 ]
 
+MASSCAN = [
+    {
+        "name": "install dev tools",
+        "package": {"name": ["gcc", "git", "make"], "state": "present"},
+        "become": True,
+        "ignore_errors": True,
+    },
+    {
+        "name": "Download masscan source code",
+        "git": {
+            "repo": "https://github.com/robertdavidgraham/masscan.git",
+            "dest": "#{BBOT_TEMP}/masscan",
+            "single_branch": True,
+            "version": "master",
+        },
+    },
+    {
+        "name": "Build masscan",
+        "command": {
+            "chdir": "#{BBOT_TEMP}/masscan",
+            "cmd": "make -j",
+            "creates": "#{BBOT_TEMP}/masscan/bin/masscan",
+        },
+    },
+    {
+        "name": "Install masscan",
+        "copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
+    },
+]
+
 # shared module dependencies -- ffuf, massdns, chromium, etc.
 SHARED_DEPS = {}
 for var, val in list(locals().items()):
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 9f39598d2..e3ac0ec0c 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -63,7 +63,7 @@ async def setup(self):
                 self.hugewarning(
                     f"Selected target ({target_len:,} hosts) is too large, skipping IP_RANGE --> IP_ADDRESS speculation"
                 )
-                self.hugewarning(f"Enabling a port scanner (nmap or masscan) module is highly recommended")
+                self.hugewarning(f'Enabling the "portscan" module is highly recommended')
             self.range_to_ip = False
 
         return True
diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py
deleted file mode 100644
index ddeb76163..000000000
--- a/bbot/modules/masscan.py
+++ /dev/null
@@ -1,278 +0,0 @@
-import json
-from contextlib import suppress
-
-from bbot.modules.templates.portscanner import portscanner
-
-
-class masscan(portscanner):
-    flags = ["active", "portscan", "aggressive"]
-    watched_events = ["IP_ADDRESS", "IP_RANGE"]
-    produced_events = ["OPEN_TCP_PORT"]
-    meta = {"description": "Port scan with masscan. By default, scans top 100 ports."}
-    options = {
-        "top_ports": 100,
-        "ports": "",
-        # ping scan at 600 packets/s ~= entire private IP space in 8 hours
-        "rate": 600,
-        "wait": 5,
-        "ping_first": False,
-        "ping_only": False,
-        "use_cache": False,
-    }
-    options_desc = {
-        "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
-        "ports": "Ports to scan",
-        "rate": "Rate in packets per second",
-        "wait": "Seconds to wait for replies after scan is complete",
-        "ping_first": "Only portscan hosts that reply to pings",
-        "ping_only": "Ping sweep only, no portscan",
-        "use_cache": "Instead of scanning, use the results from the previous scan",
-    }
-    deps_ansible = [
-        {
-            "name": "install dev tools",
-            "package": {"name": ["gcc", "git", "make"], "state": "present"},
-            "become": True,
-            "ignore_errors": True,
-        },
-        {
-            "name": "Download masscan source code",
-            "git": {
-                "repo": "https://github.com/robertdavidgraham/masscan.git",
-                "dest": "#{BBOT_TEMP}/masscan",
-                "single_branch": True,
-                "version": "master",
-            },
-        },
-        {
-            "name": "Build masscan",
-            "command": {
-                "chdir": "#{BBOT_TEMP}/masscan",
-                "cmd": "make -j",
-                "creates": "#{BBOT_TEMP}/masscan/bin/masscan",
-            },
-        },
-        {
-            "name": "Install masscan",
-            "copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
-        },
-    ]
-    batch_size = 1000000
-
-    async def setup(self):
-        self.top_ports = self.config.get("top_ports", 100)
-        self.rate = self.config.get("rate", 600)
-        self.wait = self.config.get("wait", 10)
-        self.ping_first = self.config.get("ping_first", False)
-        self.ping_only = self.config.get("ping_only", False)
-        self.use_cache = self.config.get("use_cache", False)
-        self.ports = self.config.get("ports", "")
-        if self.ports:
-            try:
-                self.helpers.parse_port_string(self.ports)
-            except ValueError as e:
-                return False, f"Error parsing ports: {e}"
-        self.alive_hosts = dict()
-
-        _, invalid_targets = self._build_targets(self.scan.target)
-        if invalid_targets > 0:
-            self.warning(
-                f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
-            )
-
-        self.run_time = self.helpers.make_date()
-        self.ping_cache = self.scan.home / f"masscan_ping.txt"
-        self.syn_cache = self.scan.home / f"masscan_syn.txt"
-        if self.use_cache:
-            files_exist = self.ping_cache.is_file() or self.syn_cache.is_file()
-            files_empty = self.helpers.filesize(self.ping_cache) == 0 and self.helpers.filesize(self.syn_cache) == 0
-            if not files_exist:
-                return (
-                    False,
-                    f"use_cache is True but could not find cache file at {self.ping_cache} or {self.syn_cache}",
-                )
-            if files_empty:
-                return (
-                    False,
-                    f"use_cache is True but could cached files {self.ping_cache} and {self.syn_cache} are empty",
-                )
-        else:
-            self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
-        self.ping_cache_fd = None
-        self.syn_cache_fd = None
-
-        return await super().setup()
-
-    async def handle_batch(self, *events):
-        if self.use_cache:
-            await self.emit_from_cache()
-        else:
-            targets = [str(e.data) for e in events]
-            if not targets:
-                self.warning("No targets specified")
-                return
-
-            # ping scan
-            if self.ping_first or self.ping_only:
-                self.verbose("Starting masscan (ping scan)")
-
-                await self.masscan(targets, result_callback=self.append_alive_host, ping=True)
-                targets = ",".join(str(h) for h in self.alive_hosts)
-                if not targets:
-                    self.warning("No hosts responded to pings")
-                    return
-
-            # TCP SYN scan
-            if not self.ping_only:
-                self.verbose("Starting masscan (TCP SYN scan)")
-                await self.masscan(targets, result_callback=self.emit_open_tcp_port)
-            else:
-                self.verbose("Only ping sweep was requested, skipping TCP SYN scan")
-            # save memory
-            self.alive_hosts.clear()
-
-    async def masscan(self, targets, result_callback, ping=False):
-        target_file = self.helpers.tempfile(targets, pipe=False)
-        command = self._build_masscan_command(target_file, ping=ping)
-        stats_file = self.helpers.tempfile_tail(callback=self.verbose)
-        try:
-            with open(stats_file, "w") as stats_fh:
-                async for line in self.run_process_live(command, sudo=True, stderr=stats_fh):
-                    await self.process_output(line, result_callback=result_callback)
-        finally:
-            for file in (stats_file, target_file):
-                file.unlink()
-
-    def _build_masscan_command(self, target_file=None, dry_run=False, ping=False):
-        command = (
-            "masscan",
-            "--excludefile",
-            str(self.exclude_file),
-            "--rate",
-            self.rate,
-            "--wait",
-            self.wait,
-            "--open-only",
-            "-oJ",
-            "-",
-        )
-        if target_file is not None:
-            command += ("-iL", str(target_file))
-        if ping:
-            command += ("--ping",)
-        else:
-            if self.ports:
-                command += ("-p", self.ports)
-            else:
-                command += ("--top-ports", str(self.top_ports))
-        if dry_run:
-            command += ("--echo",)
-        return command
-
-    async def process_output(self, line, result_callback):
-        try:
-            j = json.loads(line)
-        except Exception:
-            return
-        ip = j.get("ip", "")
-        if not ip:
-            return
-        ports = j.get("ports", [])
-        if not ports:
-            return
-        for p in ports:
-            proto = p.get("proto", "")
-            port_number = p.get("port", "")
-            if proto == "" or port_number == "":
-                continue
-            result = str(ip)
-            source = None
-            with suppress(KeyError):
-                source = self.alive_hosts[ip]
-            if proto != "icmp":
-                result = self.helpers.make_netloc(result, port_number)
-                if source is None:
-                    source = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip))
-                    if not source:
-                        continue
-                    await self.emit_event(source)
-            await result_callback(result, source=source)
-
-    async def append_alive_host(self, host, source):
-        host_event = self.make_event(host, "IP_ADDRESS", source=self.get_source_event(host))
-        if host_event:
-            self.alive_hosts[host] = host_event
-            self._write_ping_result(host)
-            await self.emit_event(host_event)
-
-    async def emit_open_tcp_port(self, data, source):
-        self._write_syn_result(data)
-        await self.emit_event(data, "OPEN_TCP_PORT", source=source)
-
-    async def emit_from_cache(self):
-        ip_events = {}
-        # ping scan
-        if self.ping_cache.is_file():
-            cached_pings = list(self.helpers.read_file(self.ping_cache))
-            if cached_pings:
-                self.success(f"{len(cached_pings):,} hosts loaded from previous ping scan")
-            else:
-                self.verbose(f"No hosts cached from previous ping scan")
-            for ip in cached_pings:
-                if self.scan.stopping:
-                    break
-                ip_event = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip))
-                if ip_event:
-                    ip_events[ip] = ip_event
-                    await self.emit_event(ip_event)
-        # syn scan
-        if self.syn_cache.is_file():
-            cached_syns = list(self.helpers.read_file(self.syn_cache))
-            if cached_syns:
-                self.success(f"{len(cached_syns):,} hosts loaded from previous SYN scan")
-            else:
-                self.warning(f"No hosts cached from previous SYN scan")
-            for line in cached_syns:
-                if self.scan.stopping:
-                    break
-                host, port = self.helpers.split_host_port(line)
-                host = str(host)
-                source_event = ip_events.get(host)
-                if source_event is None:
-                    self.verbose(f"Source event not found for {line}")
-                    source_event = self.make_event(line, "IP_ADDRESS", source=self.get_source_event(line))
-                    if not source_event:
-                        continue
-                    await self.emit_event(source_event)
-                await self.emit_event(line, "OPEN_TCP_PORT", source=source_event)
-
-    def get_source_event(self, host):
-        source_event = self.scan.target.get(host)
-        if source_event is None:
-            source_event = self.scan.whitelist.get(host)
-            if source_event is None:
-                source_event = self.scan.root_event
-        return source_event
-
-    async def cleanup(self):
-        if self.ping_first:
-            with suppress(Exception):
-                self.ping_cache_fd.close()
-        with suppress(Exception):
-            self.syn_cache_fd.close()
-        with suppress(Exception):
-            self.exclude_file.unlink()
-
-    def _write_ping_result(self, host):
-        if self.ping_cache_fd is None:
-            self.helpers.backup_file(self.ping_cache)
-            self.ping_cache_fd = open(self.ping_cache, "w")
-        self.ping_cache_fd.write(f"{host}\n")
-        self.ping_cache_fd.flush()
-
-    def _write_syn_result(self, data):
-        if self.syn_cache_fd is None:
-            self.helpers.backup_file(self.syn_cache)
-            self.syn_cache_fd = open(self.syn_cache, "w")
-        self.syn_cache_fd.write(f"{data}\n")
-        self.syn_cache_fd.flush()
diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py
deleted file mode 100644
index 4374155ab..000000000
--- a/bbot/modules/nmap.py
+++ /dev/null
@@ -1,138 +0,0 @@
-from lxml import etree
-from bbot.modules.templates.portscanner import portscanner
-
-
-class nmap(portscanner):
-    watched_events = ["IP_ADDRESS", "DNS_NAME", "IP_RANGE"]
-    produced_events = ["OPEN_TCP_PORT"]
-    flags = ["active", "portscan", "aggressive", "web-thorough"]
-    meta = {"description": "Port scan with nmap. By default, scans top 100 ports."}
-    options = {
-        "top_ports": 100,
-        "ports": "",
-        "timing": "T4",
-        "skip_host_discovery": True,
-    }
-    options_desc = {
-        "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
-        "ports": "Ports to scan",
-        "timing": "-T<0-5>: Set timing template (higher is faster)",
-        "skip_host_discovery": "skip host discovery (-Pn)",
-    }
-    _max_event_handlers = 2
-    batch_size = 256
-    _priority = 2
-
-    deps_apt = ["nmap"]
-    deps_pip = ["lxml~=4.9.2"]
-
-    async def setup(self):
-        self.helpers.depsinstaller.ensure_root(message="Nmap requires root privileges")
-        self.ports = self.config.get("ports", "")
-        self.timing = self.config.get("timing", "T4")
-        self.top_ports = self.config.get("top_ports", 100)
-        self.skip_host_discovery = self.config.get("skip_host_discovery", True)
-        return await super().setup()
-
-    async def handle_batch(self, *events):
-        target = self.helpers.make_target(*events)
-        targets = list(set(str(e.data) for e in events))
-        command, output_file = self.construct_command(targets)
-        try:
-            await self.run_process(command, sudo=True)
-            for host in self.parse_nmap_xml(output_file):
-                source_event = None
-                for h in [host.address] + host.hostnames:
-                    source_event = target.get_host(h)
-                    if source_event is not None:
-                        break
-                if source_event is None:
-                    self.warning(f"Failed to correlate source event from {host}")
-                    source_event = self.scan.root_event
-                for port in host.open_ports:
-                    port_number = int(port.split("/")[0])
-                    netloc = self.helpers.make_netloc(host.address, port_number)
-                    await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event)
-                    for hostname in host.hostnames:
-                        netloc = self.helpers.make_netloc(hostname, port_number)
-                        await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event)
-        finally:
-            output_file.unlink(missing_ok=True)
-
-    def construct_command(self, targets):
-        ports = self.config.get("ports", "")
-        top_ports = self.config.get("top_ports", "")
-        temp_filename = self.helpers.temp_filename(extension="xml")
-        command = [
-            "nmap",
-            "--noninteractive",
-            "--excludefile",
-            str(self.exclude_file),
-            "-n",
-            "--resolve-all",
-            f"-{self.timing}",
-            "-oX",
-            temp_filename,
-        ]
-        if self.skip_host_discovery:
-            command += ["-Pn"]
-        if ports:
-            command += ["-p", ports]
-        else:
-            command += ["--top-ports", top_ports]
-        command += targets
-        return command, temp_filename
-
-    def parse_nmap_xml(self, xml_file):
-        try:
-            with open(xml_file, "rb") as f:
-                et = etree.parse(f)
-                for host in et.iter("host"):
-                    yield NmapHost(host)
-        except Exception as e:
-            self.warning(f"Error parsing Nmap XML at {xml_file}: {e}")
-
-    async def cleanup(self):
-        resume_file = self.helpers.current_dir / "resume.cfg"
-        resume_file.unlink(missing_ok=True)
-
-
-class NmapHost(str):
-    def __init__(self, xml):
-        self.etree = xml
-
-        # convenient host information
-        self.status = self.etree.find("status").attrib.get("state", "down")
-        self.address = self.etree.find("address").attrib.get("addr", "")
-        self.hostnames = []
-        for hostname in self.etree.findall("hostnames/hostname"):
-            hostname = hostname.attrib.get("name")
-            if hostname and not hostname in self.hostnames:
-                self.hostnames.append(hostname)
-
-        # convenient port information
-        self.scripts = dict()
-        self.open_ports = []
-        self.closed_ports = []
-        self.filtered_ports = []
-        for port in self.etree.findall("ports/port"):
-            port_name = port.attrib.get("portid", "0") + "/" + port.attrib.get("protocol", "tcp").lower()
-            port_status = port.find("state").attrib.get("state", "closed")
-            if port_status in ("open", "closed", "filtered"):
-                getattr(self, f"{port_status}_ports").append(port_name)
-            for script in port.iter("script"):
-                script_name = script.attrib.get("id", "")
-                script_output = script.attrib.get("output", "")
-                if script_name:
-                    try:
-                        self.scripts[port_name][script_name] = script_output
-                    except KeyError:
-                        self.scripts[port_name] = {script_name: script_output}
-
-    def __str__(self):
-        address = self.address + (" " if self.address else "")
-        hostnames = "(" + ", ".join(self.hostnames) + ")" if self.hostnames else ""
-        return f"{address}{hostnames}"
-
-    def __repr__(self):
-        return str(self)
diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py
index b39ce0896..525c4d0c3 100644
--- a/bbot/scanner/preset/args.py
+++ b/bbot/scanner/preset/args.py
@@ -28,7 +28,7 @@ class BBOTArgs:
         (
             "Subdomains + port scan + web screenshots",
             "Port-scan every subdomain, screenshot every webpage, output to current directory",
-            "bbot -t evilcorp.com -p subdomain-enum -m nmap gowitness -n my_scan -o .",
+            "bbot -t evilcorp.com -p subdomain-enum -m portscan gowitness -n my_scan -o .",
         ),
         (
             "Subdomains + basic web scan",
diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index 4f4138a06..09f7cc879 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -299,15 +299,15 @@ def merge(self, other):
             other (Preset): The preset to merge into this one.
 
         Examples:
-            >>> preset1 = Preset(modules=["nmap"])
+            >>> preset1 = Preset(modules=["portscan"])
             >>> preset1.scan_modules
-            ['nmap']
+            ['portscan']
             >>> preset2 = Preset(modules=["sslcert"])
             >>> preset2.scan_modules
             ['sslcert']
             >>> preset1.merge(preset2)
             >>> preset1.scan_modules
-            ['nmap', 'sslcert']
+            ['portscan', 'sslcert']
         """
         self.log_debug(f'Merging preset "{other.name}" into "{self.name}"')
         # config
@@ -644,7 +644,7 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
             Preset: The loaded preset
 
         Examples:
-            >>> preset = Preset.from_dict({"target": "evilcorp.com", "modules": ["nmap}]})
+            >>> preset = Preset.from_dict({"target": "evilcorp.com", "modules": ["portscan}]})
         """
         new_preset = cls(
             *preset_dict.get("target", []),
@@ -727,9 +727,9 @@ def to_dict(self, include_target=False, full_config=False):
             dict: The preset in dictionary form
 
         Examples:
-            >>> preset = Preset(flags=["subdomain-enum"], modules=["nmap"])
+            >>> preset = Preset(flags=["subdomain-enum"], modules=["portscan"])
             >>> preset.to_dict()
-            {"flags": ["subdomain-enum"], "modules": ["nmap"]}
+            {"flags": ["subdomain-enum"], "modules": ["portscan"]}
         """
         preset_dict = {}
 
@@ -803,12 +803,12 @@ def to_yaml(self, include_target=False, full_config=False, sort_keys=False):
             str: The preset in the form of a YAML string
 
         Examples:
-            >>> preset = Preset(flags=["subdomain-enum"], modules=["nmap"])
+            >>> preset = Preset(flags=["subdomain-enum"], modules=["portscan"])
             >>> print(preset.to_yaml())
             flags:
             - subdomain-enum
             modules:
-            - nmap
+            - portscan
         """
         preset_dict = self.to_dict(include_target=include_target, full_config=full_config)
         return yaml.dump(preset_dict, sort_keys=sort_keys)
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index ee17ef6a7..8d032161a 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -26,11 +26,11 @@ class Scanner:
 
     Examples:
         Create scan with multiple targets:
-        >>> my_scan = Scanner("evilcorp.com", "1.2.3.0/24", modules=["nmap", "sslcert", "httpx"])
+        >>> my_scan = Scanner("evilcorp.com", "1.2.3.0/24", modules=["portscan", "sslcert", "httpx"])
 
         Create scan with custom config:
-        >>> config = {"http_proxy": "http://127.0.0.1:8080", "modules": {"nmap": {"top_ports": 2000}}}
-        >>> my_scan = Scanner("www.evilcorp.com", modules=["nmap", "httpx"], config=config)
+        >>> config = {"http_proxy": "http://127.0.0.1:8080", "modules": {"portscan": {"top_ports": 2000}}}
+        >>> my_scan = Scanner("www.evilcorp.com", modules=["portscan", "httpx"], config=config)
 
         Start the scan, iterating over events as they're discovered (synchronous):
         >>> for event in my_scan.start():
diff --git a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
index 6b6c78dbf..c9703ea92 100644
--- a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
+++ b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
@@ -4,8 +4,8 @@
 class TestAsset_Inventory(ModuleTestBase):
     targets = ["127.0.0.1", "bbottest.notreal"]
     scan_name = "asset_inventory_test"
-    config_overrides = {"dns_resolution": True, "modules": {"nmap": {"ports": "9999"}}}
-    modules_overrides = ["asset_inventory", "nmap", "sslcert"]
+    config_overrides = {"dns_resolution": True, "modules": {"portscan": {"ports": "9999"}}}
+    modules_overrides = ["asset_inventory", "portscan", "sslcert"]
 
     async def setup_before_prep(self, module_test):
         await module_test.mock_dns(
diff --git a/bbot/test/test_step_2/module_tests/test_module_masscan.py b/bbot/test/test_step_2/module_tests/test_module_masscan.py
deleted file mode 100644
index c489d8518..000000000
--- a/bbot/test/test_step_2/module_tests/test_module_masscan.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestMasscan(ModuleTestBase):
-    targets = ["8.8.8.8/32"]
-    scan_name = "test_masscan"
-    config_overrides = {"modules": {"masscan": {"ports": "443", "wait": 1}}}
-
-    masscan_output = """[
-{   "ip": "8.8.8.8",   "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }
-]"""
-
-    async def setup_after_prep(self, module_test):
-        self.masscan_run = False
-
-        async def run_masscan(command, *args, **kwargs):
-            if "masscan" in command[:2]:
-                for l in self.masscan_output.splitlines():
-                    yield l
-                self.masscan_run = True
-            else:
-                async for l in module_test.scan.helpers.run_live(command, *args, **kwargs):
-                    yield l
-
-        module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", run_masscan)
-
-    def check(self, module_test, events):
-        assert self.masscan_run == True
-        assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted"
-        assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted"
-
-
-class TestMasscan1(TestMasscan):
-    modules_overrides = ["masscan"]
-    config_overrides = {"modules": {"masscan": {"ports": "443", "wait": 1, "use_cache": True}}}
-
-    def check(self, module_test, events):
-        assert self.masscan_run == False
-        assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted"
-        assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted"
diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap.py b/bbot/test/test_step_2/module_tests/test_module_nmap.py
deleted file mode 100644
index 32b5495e5..000000000
--- a/bbot/test/test_step_2/module_tests/test_module_nmap.py
+++ /dev/null
@@ -1,79 +0,0 @@
-from .base import ModuleTestBase
-
-
-class TestNmap(ModuleTestBase):
-    targets = ["127.0.0.1/31"]
-    config_overrides = {"modules": {"nmap": {"ports": "8888,8889"}}}
-
-    async def setup_after_prep(self, module_test):
-        # make sure our IP_RANGE / IP_ADDRESS filtering is working right
-        # IPs within the target IP range should be rejected
-        ip_event_1 = module_test.scan.make_event("127.0.0.0", source=module_test.scan.root_event)
-        ip_event_1.scope_distance = 0
-        ip_event_1_result = await module_test.module._event_postcheck(ip_event_1)
-        assert ip_event_1_result[0] == False
-        assert (
-            "it did not meet custom filter criteria: skipping 127.0.0.0 because it is already included in 127.0.0.0/31"
-            in ip_event_1_result[1]
-        )
-        # but ones outside should be accepted
-        ip_event_2 = module_test.scan.make_event("127.0.0.3", source=module_test.scan.root_event)
-        ip_event_2.scope_distance = 0
-        assert (await module_test.module._event_postcheck(ip_event_2))[0] == True
-
-    def check(self, module_test, events):
-        assert 1 == len([e for e in events if e.data == "127.0.0.1:8888"])
-        assert not any(e.data == "127.0.0.1:8889" for e in events)
-
-
-class TestNmapAssetInventory(ModuleTestBase):
-    targets = ["127.0.0.1/31"]
-    config_overrides = {
-        "modules": {"nmap": {"ports": "8888,8889"}, "asset_inventory": {"use_previous": True}},
-    }
-    modules_overrides = ["nmap", "asset_inventory"]
-    module_name = "nmap"
-    scan_name = "nmap_test_asset_inventory"
-
-    async def setup_after_prep(self, module_test):
-        from bbot.scanner import Scanner
-
-        first_scan = Scanner(
-            "127.0.0.1",
-            scan_name=self.scan_name,
-            output_modules=["asset_inventory"],
-            config={
-                "modules": {"nmap": {"ports": "8888,8889"}, "asset_inventory": {"use_previous": False}},
-            },
-        )
-        await first_scan.async_start_without_generator()
-
-        asset_inventory_output_file = first_scan.home / "asset-inventory.csv"
-        assert "127.0.0.1," in open(asset_inventory_output_file).read()
-        # make sure our IP_RANGE / IP_ADDRESS filtering is working right
-        # IPs within the target IP range should not be rejected because asset_inventory.use_previous=true
-        ip_event_1 = module_test.scan.make_event("127.0.0.0", source=module_test.scan.root_event)
-        ip_event_1.scope_distance = 0
-        assert (await module_test.module._event_postcheck(ip_event_1))[0] == True
-        # but ones outside should be accepted
-        ip_event_2 = module_test.scan.make_event("127.0.0.3", source=module_test.scan.root_event)
-        ip_event_2.scope_distance = 0
-        assert (await module_test.module._event_postcheck(ip_event_2))[0] == True
-
-        ip_range_event = module_test.scan.make_event("127.0.0.1/31", source=module_test.scan.root_event)
-        ip_range_event.scope_distance = 0
-        ip_range_filter_result = await module_test.module._event_postcheck(ip_range_event)
-        assert ip_range_filter_result[0] == False
-        assert f"skipping IP_RANGE 127.0.0.0/31 because asset_inventory.use_previous=True" in ip_range_filter_result[1]
-
-    def check(self, module_test, events):
-        assert 1 == len(
-            [
-                e
-                for e in events
-                if e.data == "127.0.0.1:8888"
-                and e.source.data == "127.0.0.1"
-                and str(e.source.module) == "asset_inventory"
-            ]
-        )
-        assert not any(e.data == "127.0.0.1:8889" for e in events)
diff --git a/bbot/test/test_step_2/module_tests/test_module_speculate.py b/bbot/test/test_step_2/module_tests/test_module_speculate.py
index 2f7d6b7f3..8b6150919 100644
--- a/bbot/test/test_step_2/module_tests/test_module_speculate.py
+++ b/bbot/test/test_step_2/module_tests/test_module_speculate.py
@@ -71,7 +71,7 @@ def check(self, module_test, events):
 
 class TestSpeculate_OpenPorts_Portscanner(TestSpeculate_OpenPorts):
     targets = ["evilcorp.com"]
-    modules_overrides = ["speculate", "certspotter", "nmap"]
+    modules_overrides = ["speculate", "certspotter", "portscan"]
     config_overrides = {"speculate": True}
 
     def check(self, module_test, events):
diff --git a/docs/dev/helpers/command.md b/docs/dev/helpers/command.md
index b2b9171f5..3716d2037 100644
--- a/docs/dev/helpers/command.md
+++ b/docs/dev/helpers/command.md
@@ -1,6 +1,6 @@
 # Command Helpers
 
-These are helpers related to executing shell commands. They are used throughout BBOT and its modules for executing various binaries such as `nmap`, `nuclei`, etc.
+These are helpers related to executing shell commands. They are used throughout BBOT and its modules for executing various binaries such as `masscan`, `nuclei`, etc.
 
 These helpers can be invoked directly from `self.helpers`, but inside a module they should always use `self.run_process()` or `self.run_process_live()`. These are light wrappers which ensure the running process is tracked by the module so that it can be easily terminated should the user need to kill the module:
 
diff --git a/docs/how_it_works.md b/docs/how_it_works.md
index d16bb50cf..aa4348fc7 100644
--- a/docs/how_it_works.md
+++ b/docs/how_it_works.md
@@ -1,6 +1,6 @@
 # What is BBOT?
 
-BBOT is a system of individual modules that interchange data **recursively**. Every module (e.g. `nmap`) _consumes_ a type of data (e.g. a `DNS_NAME`) and _emits_ another kind, (an `OPEN_TCP_PORT`). These bits of data, called [events](scanning/events.md), become the output of the tool, but are also redistributed to all the other modules, prompting them to dig deeper, and feeding the recursive cycle of discovery.
+BBOT is a system of individual modules that interchange data **recursively**. Every module (e.g. `portscan`) _consumes_ a type of data (e.g. a `DNS_NAME`) and _emits_ another kind, (an `OPEN_TCP_PORT`). These bits of data, called [events](scanning/events.md), become the output of the tool, but are also redistributed to all the other modules, prompting them to dig deeper, and feeding the recursive cycle of discovery.
 
 ![recursion](https://github.com/blacklanternsecurity/bbot/assets/20261699/7b2edfca-2692-463b-939b-ab9d52d2fe00)
 
@@ -24,21 +24,21 @@ Recursion is at the heart of BBOT's design. Each newly-discovered piece of data
 
 ## Module Example
 
-In a simple example, we run a BBOT scan with **three modules**: `nmap`, `sslcert`, and `httpx`. Each of these modules "consume" a certain type of data:
+In a simple example, we run a BBOT scan with **three modules**: `portscan`, `sslcert`, and `httpx`. Each of these modules "consume" a certain type of data:
 
-- **`nmap`** consumes `DNS_NAME`s, port-scans them, and outputs `OPEN_TCP_PORT`s
+- **`portscan`** consumes `DNS_NAME`s, port-scans them, and outputs `OPEN_TCP_PORT`s
 - **`sslcert`** consumes `OPEN_TCP_PORT`s, grabs certs, and extracts `DNS_NAME`s
 - **`httpx`** consumes `OPEN_TCP_PORT`s and visits any web services, ultimately producing new `DNS_NAME`s
 
 ```mermaid
 graph TD
-  nmap -->|OPEN_TCP_PORT| sslcert;
-  nmap -->|OPEN_TCP_PORT| httpx;
-  sslcert --> |DNS_NAME| nmap;
-  httpx --> |DNS_NAME| nmap;
+  portscan -->|OPEN_TCP_PORT| sslcert;
+  portscan -->|OPEN_TCP_PORT| httpx;
+  sslcert --> |DNS_NAME| portscan;
+  httpx --> |DNS_NAME| portscan;
 ```
 
-This allows for some interesting chains of events. Given a single target such as `evilcorp.com`, `nmap` may start by discovering an `OPEN_TCP_PORT` `evilcorp.com:443`. `sslcert` and `httpx` will then visit that port and extract more hostnames, which are in turn scanned by `nmap` to produce more open ports which are visited by `sslcert` and `httpx`, which discover more hostnames, which are again passed to `nmap`, and so on...
+This allows for some interesting chains of events. Given a single target such as `evilcorp.com`, `portscan` may start by discovering an `OPEN_TCP_PORT` `evilcorp.com:443`. `sslcert` and `httpx` will then visit that port and extract more hostnames, which are in turn scanned by `portscan` to produce more open ports which are visited by `sslcert` and `httpx`, which discover more hostnames, which are again passed to `portscan`, and so on...
 
 This is a simple example with only a few modules, but you can being to see how if 30 or 40 modules were enabled, they could feed each other exponentially to produce an immense amount of data. This recursion is exactly how BBOT is able to outperform other tools.
 
diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md
index 138b2b6ee..f0f3efed6 100644
--- a/docs/modules/nuclei.md
+++ b/docs/modules/nuclei.md
@@ -103,20 +103,20 @@ The **ratelimit** and **concurrency** settings default to the same defaults that
 
 ```bash
 # Scan a SINGLE target with a basic port scan and web modules
-bbot -f web-basic -m nmap nuclei --allow-deadly -t app.evilcorp.com
+bbot -f web-basic -m portscan nuclei --allow-deadly -t app.evilcorp.com
 ```
 
 ```bash
 # Scanning MULTIPLE targets
-bbot -f web-basic -m nmap nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
+bbot -f web-basic -m portscan nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
 ```
 
 ```bash
 # Scanning MULTIPLE targets while performing subdomain enumeration
-bbot -f subdomain-enum web-basic -m nmap nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
+bbot -f subdomain-enum web-basic -m portscan nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
 ```
 
 ```bash
 # Scanning MULTIPLE targets on a BUDGET
-bbot -f subdomain-enum web-basic -m nmap nuclei --allow-deadly -c modules.nuclei.mode=budget -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
+bbot -f subdomain-enum web-basic -m portscan nuclei --allow-deadly -c modules.nuclei.mode=budget -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
 ```
diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md
index c416d6350..9e85ac84f 100644
--- a/docs/scanning/advanced.md
+++ b/docs/scanning/advanced.md
@@ -10,7 +10,7 @@ Below you can find some advanced uses of BBOT.
 from bbot.scanner import Scanner
 
 # any number of targets can be specified
-scan = Scanner("example.com", "scanme.nmap.org", modules=["nmap", "sslcert"])
+scan = Scanner("example.com", "scanme.nmap.org", modules=["portscan", "sslcert"])
 for event in scan.start():
     print(event.json())
 ```
@@ -21,7 +21,7 @@ for event in scan.start():
 from bbot.scanner import Scanner
 
 async def main():
-    scan = Scanner("example.com", "scanme.nmap.org", modules=["nmap", "sslcert"])
+    scan = Scanner("example.com", "scanme.nmap.org", modules=["portscan", "sslcert"])
     async for event in scan.async_start():
         print(event.json())
 
diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index fafedba1e..5e370f90b 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -212,7 +212,7 @@ dns_omit_queries:
 
 ## Module Config Options
 
-Many modules accept their own configuration options. These options have the ability to change their behavior. For example, the `nmap` module accepts options for `ports`, `timing`, etc. Below is a list of all possible module config options.
+Many modules accept their own configuration options. These options have the ability to change their behavior. For example, the `portscan` module accepts options for `ports`, `rate`, etc. Below is a list of all possible module config options.
 
 <!-- BBOT MODULE OPTIONS -->
 | Config Option                                  | Type   | Description                                                                                                                                                                                                                                                                                                                    | Default                                                                                                                                                                                                                                                                                                                                                                           |

From 6ae6b8a17e25cc482a972d75dc6e41b025b548cd Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 14 May 2024 16:48:30 -0400
Subject: [PATCH 134/220] remove portscanner template

---
 bbot/modules/templates/portscanner.py | 55 ---------------------------
 1 file changed, 55 deletions(-)
 delete mode 100644 bbot/modules/templates/portscanner.py

diff --git a/bbot/modules/templates/portscanner.py b/bbot/modules/templates/portscanner.py
deleted file mode 100644
index 5d1662d81..000000000
--- a/bbot/modules/templates/portscanner.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import ipaddress
-
-from bbot.modules.base import BaseModule
-
-
-class portscanner(BaseModule):
-    """
-    A portscanner containing useful methods for nmap, masscan, etc.
-    """
-
-    async def setup(self):
-        self.ip_ranges = [e.host for e in self.scan.target.events if e.type == "IP_RANGE"]
-        exclude, invalid_exclude = self._build_targets(self.scan.blacklist)
-        if not exclude:
-            exclude = ["255.255.255.255/32"]
-        self.exclude_file = self.helpers.tempfile(exclude, pipe=False)
-        if invalid_exclude > 0:
-            self.warning(
-                f"Port scanner can only accept IP addresses or IP ranges as blacklist ({invalid_exclude:,} blacklisted were hostnames)"
-            )
-        return True
-
-    async def filter_event(self, event):
-        """
-        The purpose of this filter_event is to decide whether we should accept individual IP_ADDRESS
-        events that reside inside our target subnets (IP_RANGE), if any.
-
-        This prevents scanning the same IP twice.
-        """
-        # if we are emitting hosts from a previous asset_inventory, this is a special case
-        # in this case we want to accept the individual IPs even if they overlap with our target ranges
-        asset_inventory_module = self.scan.modules.get("asset_inventory", None)
-        asset_inventory_config = getattr(asset_inventory_module, "config", {})
-        asset_inventory_use_previous = asset_inventory_config.get("use_previous", False)
-        if event.type == "IP_ADDRESS" and not asset_inventory_use_previous:
-            for net in self.helpers.ip_network_parents(event.data, include_self=True):
-                if net in self.ip_ranges:
-                    return False, f"skipping {event.host} because it is already included in {net}"
-        elif event.type == "IP_RANGE" and asset_inventory_use_previous:
-            return False, f"skipping IP_RANGE {event.host} because asset_inventory.use_previous=True"
-        return True
-
-    def _build_targets(self, target):
-        invalid_targets = 0
-        targets = []
-        for t in target:
-            t = self.helpers.make_ip_type(t.data)
-            if isinstance(t, str):
-                invalid_targets += 1
-            else:
-                if self.helpers.is_ip(t):
-                    targets.append(str(ipaddress.ip_network(t)))
-                else:
-                    targets.append(str(t))
-        return targets, invalid_targets

From 387f6088992b9b49f810cdae076935d4d08633ce Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 10:07:51 -0400
Subject: [PATCH 135/220] better ctrl+c error handling

---
 bbot/core/engine.py                      | 4 ++--
 bbot/modules/templates/subdomain_enum.py | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index 38e9cd30c..a480e1c68 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -210,7 +210,7 @@ async def run_and_return(self, client_id, command_fn, *args, **kwargs):
             try:
                 result = await command_fn(*args, **kwargs)
             except (asyncio.CancelledError, KeyboardInterrupt):
-                pass
+                return
             except BaseException as e:
                 error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
                 trace = traceback.format_exc()
@@ -234,7 +234,7 @@ async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
                     await self.send_socket_multipart(client_id, _)
                 await self.send_socket_multipart(client_id, {"_s": None})
             except (asyncio.CancelledError, KeyboardInterrupt):
-                pass
+                return
             except BaseException as e:
                 error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
                 trace = traceback.format_exc()
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 44d4e6a50..a7e5ada89 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -70,7 +70,6 @@ def parse_results(self, r, query=None):
                 yield hostname
 
     async def query(self, query, parse_fn=None, request_fn=None):
-        self.hugesuccess(query)
         if parse_fn is None:
             parse_fn = self.parse_results
         if request_fn is None:

From 584bf317be48a714cc65f77b49a9b278f9ebda4d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 12:44:10 -0400
Subject: [PATCH 136/220] implement process timeouts, implement in gowitness

---
 bbot/core/helpers/command.py          | 14 ++++++++----
 bbot/modules/gowitness.py             | 32 +++++++++++++++++++--------
 bbot/test/test_step_1/test_command.py | 18 +++++++++++++++
 3 files changed, 51 insertions(+), 13 deletions(-)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index 06fc8a91f..ab321897a 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -9,7 +9,7 @@
 log = logging.getLogger("bbot.core.helpers.command")
 
 
-async def run(self, *command, check=False, text=True, **kwargs):
+async def run(self, *command, check=False, text=True, idle_timeout=None, **kwargs):
     """Runs a command asynchronously and gets its output as a string.
 
     This method is a simple helper for executing a command and capturing its output.
@@ -45,7 +45,10 @@ async def run(self, *command, check=False, text=True, **kwargs):
                     _input = b"\n".join(smart_encode(i) for i in _input) + b"\n"
                 else:
                     _input = smart_encode(_input)
-            stdout, stderr = await proc.communicate(_input)
+            if idle_timeout is not None:
+                stdout, stderr = await asyncio.wait_for(proc.communicate(_input), timeout=idle_timeout)
+            else:
+                stdout, stderr = await proc.communicate(_input)
 
             # surface stderr
             if text:
@@ -65,7 +68,7 @@ async def run(self, *command, check=False, text=True, **kwargs):
             proc_tracker.remove(proc)
 
 
-async def run_live(self, *command, check=False, text=True, **kwargs):
+async def run_live(self, *command, check=False, text=True, idle_timeout=None, **kwargs):
     """Runs a command asynchronously and iterates through its output line by line in realtime.
 
     This method is useful for executing a command and capturing its output on-the-fly, as it is generated.
@@ -102,7 +105,10 @@ async def run_live(self, *command, check=False, text=True, **kwargs):
 
             while 1:
                 try:
-                    line = await proc.stdout.readline()
+                    if idle_timeout is not None:
+                        line = await asyncio.wait_for(proc.stdout.readline(), timeout=idle_timeout)
+                    else:
+                        line = await proc.stdout.readline()
                 except ValueError as e:
                     command_str = " ".join([str(c) for c in command])
                     log.warning(f"Error executing command {command_str}: {e}")
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 3271ef93f..258dce6d6 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -1,4 +1,5 @@
 import sqlite3
+import multiprocessing
 from pathlib import Path
 from contextlib import suppress
 from shutil import copyfile, copymode
@@ -19,15 +20,17 @@ class gowitness(BaseModule):
         "resolution_y": 900,
         "output_path": "",
         "social": True,
+        "idle_timeout": 1800,
     }
     options_desc = {
-        "version": "gowitness version",
-        "threads": "threads used to run",
-        "timeout": "preflight check timeout",
-        "resolution_x": "screenshot resolution x",
-        "resolution_y": "screenshot resolution y",
-        "output_path": "where to save screenshots",
+        "version": "Gowitness version",
+        "threads": "How many gowitness threads to spawn - default is number of CPUs x 2",
+        "timeout": "Preflight check timeout",
+        "resolution_x": "Screenshot resolution x",
+        "resolution_y": "Screenshot resolution y",
+        "output_path": "Where to save screenshots",
         "social": "Whether to screenshot social media webpages",
+        "idle_timeout": "Skip the current gowitness batch if it stalls for longer than this many seconds",
     }
     deps_ansible = [
         {
@@ -82,8 +85,11 @@ class gowitness(BaseModule):
     scope_distance_modifier = 2
 
     async def setup(self):
+        num_cpus = multiprocessing.cpu_count()
+        default_thread_count = min(20, num_cpus * 2)
         self.timeout = self.config.get("timeout", 10)
-        self.threads = self.config.get("threads", 4)
+        self.idle_timeout = self.config.get("idle_timeout", 1800)
+        self.threads = self.config.get("threads", default_thread_count)
         self.proxy = self.scan.config.get("http_proxy", "")
         self.resolution_x = self.config.get("resolution_x")
         self.resolution_y = self.config.get("resolution_y")
@@ -155,8 +161,13 @@ async def handle_batch(self, *events):
             event_dict[key] = e
         stdin = "\n".join(list(event_dict))
 
-        async for line in self.run_process_live(self.command, input=stdin):
-            self.debug(line)
+        try:
+            async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):
+                self.debug(line)
+        except TimeoutError as e:
+            urls_str = ",".join(event_dict)
+            self.warning(f"Gowitness timed out while visiting the following URLs: {urls_str}", trace=False)
+            return
 
         # emit web screenshots
         for filename, screenshot in self.new_screenshots.items():
@@ -191,6 +202,7 @@ async def handle_batch(self, *events):
             await self.emit_event(tech_data, "TECHNOLOGY", source=source_event)
 
     def construct_command(self):
+        return ["sleep", "999"]
         # base executable
         command = ["gowitness"]
         # chrome path
@@ -212,6 +224,8 @@ def construct_command(self):
         command += ["file", "-f", "-"]
         # threads
         command += ["--threads", str(self.threads)]
+        # timeout
+        command += ["--timeout", str(self.timeout)]
         return command
 
     @property
diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py
index 8827bcdad..0651f857a 100644
--- a/bbot/test/test_step_1/test_command.py
+++ b/bbot/test/test_step_1/test_command.py
@@ -1,3 +1,4 @@
+import time
 from ..bbot_fixtures import *
 from subprocess import CalledProcessError
 
@@ -6,6 +7,23 @@
 async def test_command(bbot_scanner, bbot_config):
     scan1 = bbot_scanner(config=bbot_config)
 
+    # test timeouts
+    command = ["sleep", "3"]
+    start = time.time()
+    with pytest.raises(TimeoutError):
+        await scan1.helpers.run(command, idle_timeout=1)
+    end = time.time()
+    elapsed = end - start
+    assert 0 < elapsed < 2
+
+    start = time.time()
+    with pytest.raises(TimeoutError):
+        async for line in scan1.helpers.run_live(command, idle_timeout=1):
+            print(line)
+    end = time.time()
+    elapsed = end - start
+    assert 0 < elapsed < 2
+
     # run
     assert "plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"])).stdout
     assert b"plumbus\n" == (await scan1.helpers.run(["echo", "plumbus"], text=False)).stdout

From 41d195a04140963dcf553ffb5dafa03d9ea6280d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 12:47:38 -0400
Subject: [PATCH 137/220] remove debugging

---
 bbot/modules/gowitness.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 258dce6d6..91f878ce4 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -202,7 +202,6 @@ async def handle_batch(self, *events):
             await self.emit_event(tech_data, "TECHNOLOGY", source=source_event)
 
     def construct_command(self):
-        return ["sleep", "999"]
         # base executable
         command = ["gowitness"]
         # chrome path

From 1ad243064cb4ae21727757c6cbbba06bef3939cb Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 12:51:48 -0400
Subject: [PATCH 138/220] update documentation

---
 bbot/core/helpers/command.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index ab321897a..785d646cb 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -20,6 +20,7 @@ async def run(self, *command, check=False, text=True, idle_timeout=None, **kwarg
         check (bool, optional): If set to True, raises an error if the subprocess exits with a non-zero status.
                                 Defaults to False.
         text (bool, optional): If set to True, decodes the subprocess output to string. Defaults to True.
+        idle_timeout (int, optional): Sets a limit on the number of seconds the process can run before throwing a TimeoutError
         **kwargs (dict): Additional keyword arguments for the subprocess.
 
     Returns:
@@ -79,6 +80,7 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
         check (bool, optional): If set to True, raises an error if the subprocess exits with a non-zero status.
                                 Defaults to False.
         text (bool, optional): If set to True, decodes the subprocess output to string. Defaults to True.
+        idle_timeout (int, optional): Sets a limit on the number of seconds the process can remain idle (no lines sent to stdout) before throwing a TimeoutError
         **kwargs (dict): Additional keyword arguments for the subprocess.
 
     Yields:

From 38bf7fecc092e0a32e7631f104500ded4949bcb4 Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 15 May 2024 18:31:47 +0100
Subject: [PATCH 139/220] Added a `CODE_REPOSITORY` module to look for code
 repository links in webpages

---
 bbot/modules/code_repository.py               | 48 +++++++++++++++
 .../test_module_code_repository.py            | 59 +++++++++++++++++++
 2 files changed, 107 insertions(+)
 create mode 100644 bbot/modules/code_repository.py
 create mode 100644 bbot/test/test_step_2/module_tests/test_module_code_repository.py

diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
new file mode 100644
index 000000000..5c55e41c3
--- /dev/null
+++ b/bbot/modules/code_repository.py
@@ -0,0 +1,48 @@
+import re
+from bbot.modules.base import BaseModule
+
+
+class code_repository(BaseModule):
+    watched_events = ["URL_UNVERIFIED"]
+    produced_events = ["CODE_REPOSITORY"]
+    meta = {"description": "Look for code repository links in webpages"}
+    flags = ["passive", "safe", "repo-enum"]
+
+    # platform name : (regex, case_sensitive)
+    code_repositories = {
+        "git": [
+            (r"github.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+            (r"gitlab.(?:com|org)/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+        ],
+        "docker": (r"hub.docker.com/r/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+", False),
+    }
+
+    scope_distance_modifier = 1
+
+    async def setup(self):
+        self.compiled_regexes = {}
+        for k, v in self.code_repositories.items():
+            if isinstance(v, list):
+                self.compiled_regexes[k] = [(re.compile(pattern), c) for pattern, c in v]
+            else:
+                pattern, c = v
+                self.compiled_regexes[k] = (re.compile(pattern), c)
+        return True
+
+    async def handle_event(self, event):
+        for platform, regexes in self.compiled_regexes.items():
+            if not isinstance(regexes, list):
+                regexes = [regexes]
+            for regex, case_sensitive in regexes:
+                for match in regex.finditer(event.data):
+                    url = match.group()
+                    if not case_sensitive:
+                        url = url.lower()
+                    repo_event = self.make_event(
+                        {"url": f"https://{url}"},
+                        "CODE_REPOSITORY",
+                        tags=platform,
+                        source=event,
+                    )
+                    repo_event.scope_distance = event.scope_distance
+                    await self.emit_event(repo_event)
diff --git a/bbot/test/test_step_2/module_tests/test_module_code_repository.py b/bbot/test/test_step_2/module_tests/test_module_code_repository.py
new file mode 100644
index 000000000..3bda459c8
--- /dev/null
+++ b/bbot/test/test_step_2/module_tests/test_module_code_repository.py
@@ -0,0 +1,59 @@
+from .base import ModuleTestBase
+
+
+class TestCodeRepository(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "excavate", "code_repository"]
+
+    async def setup_after_prep(self, module_test):
+        expect_args = {"method": "GET", "uri": "/"}
+        respond_args = {
+            "response_data": """
+            <html>
+                <a href="https://github.com/blacklanternsecurity/bbot"/>
+                <a href="https://gitlab.com/blacklanternsecurity/bbot"/>
+                <a href="https://gitlab.org/blacklanternsecurity/bbot"/>
+                <a href="https://hub.docker.com/r/blacklanternsecurity/bbot"/>
+            </html>
+            """
+        }
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+    def check(self, module_test, events):
+        assert 4 == len([e for e in events if e.type == "CODE_REPOSITORY"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://github.com/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.com/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "git" in e.tags
+                and e.data["url"] == "https://gitlab.org/blacklanternsecurity/bbot"
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "CODE_REPOSITORY"
+                and "docker" in e.tags
+                and e.data["url"] == "https://hub.docker.com/r/blacklanternsecurity/bbot"
+            ]
+        )

From 17bb5804cd44ea6e6e3927a79dfe96a446818b8d Mon Sep 17 00:00:00 2001
From: Dom Whewell <dom.whewell@sage.com>
Date: Wed, 15 May 2024 18:45:29 +0100
Subject: [PATCH 140/220] Add `repo-enum` to bbot/core/flags.py

---
 bbot/core/flags.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bbot/core/flags.py b/bbot/core/flags.py
index d8dbf8566..c6b675798 100644
--- a/bbot/core/flags.py
+++ b/bbot/core/flags.py
@@ -14,6 +14,7 @@
     "service-enum": "Identifies protocols running on open ports",
     "slow": "May take a long time to complete",
     "social-enum": "Enumerates social media",
+    "repo-enum": "Enumerates code repositories",
     "subdomain-enum": "Enumerates subdomains",
     "subdomain-hijack": "Detects hijackable subdomains",
     "web-basic": "Basic, non-intrusive web scan functionality",

From 4beebade4fad7c130acbb6ea2f3a4cf2037c8585 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 14:53:57 -0400
Subject: [PATCH 141/220] flaked

---
 bbot/modules/gowitness.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 91f878ce4..2b7f9291e 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -164,7 +164,7 @@ async def handle_batch(self, *events):
         try:
             async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):
                 self.debug(line)
-        except TimeoutError as e:
+        except TimeoutError:
             urls_str = ",".join(event_dict)
             self.warning(f"Gowitness timed out while visiting the following URLs: {urls_str}", trace=False)
             return

From 42bed2e03b1cb5dd05ddf4c3ea0e2249334f3902 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 14:56:55 -0400
Subject: [PATCH 142/220] should probably add the actual python files

---
 bbot/modules/portscan.py                      | 184 ++++++++++++++++++
 .../module_tests/test_module_portscan.py      |  30 +++
 2 files changed, 214 insertions(+)
 create mode 100644 bbot/modules/portscan.py
 create mode 100644 bbot/test/test_step_2/module_tests/test_module_portscan.py

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
new file mode 100644
index 000000000..a6fc18bfd
--- /dev/null
+++ b/bbot/modules/portscan.py
@@ -0,0 +1,184 @@
+import json
+import ipaddress
+from contextlib import suppress
+from radixtarget import RadixTarget
+
+from bbot.modules.base import BaseModule
+
+
+class portscan(BaseModule):
+    flags = ["active", "portscan"]
+    watched_events = ["IP_ADDRESS", "IP_RANGE", "DNS_NAME"]
+    produced_events = ["OPEN_TCP_PORT"]
+    meta = {"description": "Port scan with masscan. By default, scans top 100 ports."}
+    options = {
+        "top_ports": 100,
+        "ports": "",
+        # ping scan at 600 packets/s ~= private IP space in 8 hours
+        "rate": 600,
+        "wait": 5,
+        "ping_first": False,
+        "ping_only": False,
+    }
+    options_desc = {
+        "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
+        "ports": "Ports to scan",
+        "rate": "Rate in packets per second",
+        "wait": "Seconds to wait for replies after scan is complete",
+        "ping_first": "Only portscan hosts that reply to pings",
+        "ping_only": "Ping sweep only, no portscan",
+    }
+    deps_shared = ["masscan"]
+    batch_size = 1000000
+
+    async def setup(self):
+        self.top_ports = self.config.get("top_ports", 100)
+        self.rate = self.config.get("rate", 600)
+        self.wait = self.config.get("wait", 10)
+        self.ping_first = self.config.get("ping_first", False)
+        self.ping_only = self.config.get("ping_only", False)
+        self.ports = self.config.get("ports", "")
+        if self.ports:
+            try:
+                self.helpers.parse_port_string(self.ports)
+            except ValueError as e:
+                return False, f"Error parsing ports: {e}"
+        self.alive_hosts = dict()
+        self.scanned_tracker = RadixTarget()
+        self.prep_blacklist()
+        self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
+        return True
+
+    async def handle_batch(self, *events):
+        targets = [str(h) for h in self.make_targets(events)]
+
+        # ping scan
+        if self.ping_first or self.ping_only:
+            new_targets = []
+            async for alive_host, _ in self.masscan(targets, ping=True):
+                source_event = self.scanned_tracker.search(alive_host)
+                if source_event.type != "DNS_NAME":
+                    await self.emit_event(alive_host, "IP_ADDRESS", source=source_event)
+                new_targets.append(ipaddress.ip_network(alive_host, strict=False))
+            targets = new_targets
+
+        # TCP SYN scan
+        if not self.ping_only:
+            async for host, port in self.masscan(targets):
+                source_event = self.scanned_tracker.search(host)
+                if source_event.type == "DNS_NAME":
+                    host = source_event.host
+                netloc = self.helpers.make_netloc(host, port)
+                await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event)
+        else:
+            self.verbose("Only ping sweep was requested, skipping TCP SYN scan")
+
+    async def masscan(self, targets, ping=False):
+        scan_type = "ping" if ping else "SYN"
+        self.verbose(f"Starting masscan {scan_type} scan")
+        if not targets:
+            self.verbose("No targets specified, aborting.")
+            return
+
+        target_file = self.helpers.tempfile(targets, pipe=False)
+        command = self._build_masscan_command(target_file, ping=ping)
+        stats_file = self.helpers.tempfile_tail(callback=self.verbose)
+        try:
+            with open(stats_file, "w") as stats_fh:
+                async for line in self.run_process_live(command, sudo=True, stderr=stats_fh):
+                    for host, port in self.parse_json_line(line):
+                        yield host, port
+        finally:
+            for file in (stats_file, target_file):
+                file.unlink()
+
+    def _build_masscan_command(self, target_file=None, dry_run=False, ping=False):
+        command = (
+            "masscan",
+            "--excludefile",
+            str(self.exclude_file),
+            "--rate",
+            self.rate,
+            "--wait",
+            self.wait,
+            "--open-only",
+            "-oJ",
+            "-",
+        )
+        if target_file is not None:
+            command += ("-iL", str(target_file))
+        if ping:
+            command += ("--ping",)
+        else:
+            if self.ports:
+                command += ("-p", self.ports)
+            else:
+                command += ("--top-ports", str(self.top_ports))
+        if dry_run:
+            command += ("--echo",)
+        return command
+
+    def make_targets(self, events):
+        # convert events into a list of targets, skipping ones that have already been scanned
+        targets = set()
+        for e in events:
+            # skip events without host
+            if not e.host:
+                continue
+            # skip events that we already scanned
+            if self.scanned_tracker.search(e.host):
+                self.debug(f"Skipping {e.host} because it was already scanned")
+                continue
+            try:
+                # first assume it's an ip address / ip range
+                host = ipaddress.ip_network(e.host, strict=False)
+                targets.add(host)
+                self.scanned_tracker.insert(host, e)
+            except Exception:
+                # if it's a hostname, get its IPs from resolved_hosts
+                hosts = set()
+                for h in e.resolved_hosts:
+                    try:
+                        h = ipaddress.ip_network(h, strict=False)
+                        hosts.add(h)
+                    except Exception:
+                        continue
+                for h in hosts:
+                    targets.add(h)
+                    self.scanned_tracker.insert(h, e)
+        return targets
+
+    def parse_json_line(self, line):
+        try:
+            j = json.loads(line)
+        except Exception:
+            return
+        ip = j.get("ip", "")
+        if not ip:
+            return
+        ports = j.get("ports", [])
+        if not ports:
+            return
+        for p in ports:
+            proto = p.get("proto", "")
+            port_number = p.get("port", 0)
+            if proto == "" or port_number == "":
+                continue
+            yield ip, port_number
+
+    def prep_blacklist(self):
+        exclude = []
+        for t in self.scan.blacklist:
+            t = self.helpers.make_ip_type(t.data)
+            if not isinstance(t, str):
+                if self.helpers.is_ip(t):
+                    exclude.append(str(ipaddress.ip_network(t)))
+                else:
+                    exclude.append(str(t))
+        if not exclude:
+            exclude = ["255.255.255.255/32"]
+        self.exclude_file = self.helpers.tempfile(exclude, pipe=False)
+
+    async def cleanup(self):
+        with suppress(Exception):
+            self.exclude_file.unlink()
diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py
new file mode 100644
index 000000000..cae6cb20c
--- /dev/null
+++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py
@@ -0,0 +1,30 @@
+from .base import ModuleTestBase
+
+
+class TestPortscan(ModuleTestBase):
+    targets = ["8.8.8.8/32"]
+    scan_name = "test_portscan"
+    config_overrides = {"modules": {"portscan": {"ports": "443", "wait": 1}}}
+
+    masscan_output = """[
+{   "ip": "8.8.8.8",   "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }
+]"""
+
+    async def setup_after_prep(self, module_test):
+        self.masscan_run = False
+
+        async def run_masscan(command, *args, **kwargs):
+            if "masscan" in command[:2]:
+                for l in self.masscan_output.splitlines():
+                    yield l
+                self.masscan_run = True
+            else:
+                async for l in module_test.scan.helpers.run_live(command, *args, **kwargs):
+                    yield l
+
+        module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", run_masscan)
+
+    def check(self, module_test, events):
+        assert self.masscan_run == True
+        assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted"
+        assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted"

From 502b8f96521829ccefb53d76ca4b9d0e8e59632b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 15:00:59 -0400
Subject: [PATCH 143/220] bump version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 8d07dd3e5..38fd76649 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,7 +91,7 @@ extend-exclude = "(test_step_1/test_manager_*)"
 [tool.poetry-dynamic-versioning]
 enable = true
 metadata = false
-format-jinja = 'v1.1.7{% if branch == "dev" %}.{{ distance }}rc{% endif %}'
+format-jinja = 'v1.1.8{% if branch == "dev" %}.{{ distance }}rc{% endif %}'
 
 [tool.poetry-dynamic-versioning.substitution]
 files = ["*/__init__.py"]

From 1f807f5abc1d1d1fb1390ebbb04994fe09da9903 Mon Sep 17 00:00:00 2001
From: BBOT Docs Autopublish <info@blacklanternsecurity.com>
Date: Wed, 15 May 2024 19:10:50 +0000
Subject: [PATCH 144/220] Refresh module docs

---
 README.md                       | 47 +++++++++++++++++----------------
 docs/modules/list_of_modules.md |  1 +
 docs/scanning/advanced.md       |  4 +--
 docs/scanning/events.md         |  4 +--
 docs/scanning/index.md          | 47 +++++++++++++++++----------------
 5 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/README.md b/README.md
index 3fab9b285..119119abe 100644
--- a/README.md
+++ b/README.md
@@ -264,29 +264,30 @@ BBOT consistently finds 20-50% more subdomains than other tools. The bigger the
 For a full list of modules, including the data types consumed and emitted by each one, see [List of Modules](https://www.blacklanternsecurity.com/bbot/modules/list_of_modules/).
 
 <!-- BBOT MODULE FLAGS -->
-| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
-|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| safe             | 81          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
-| passive          | 61          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
-| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                              |
-| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
-| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+|------------------|-------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| safe             | 82          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
+| passive          | 62          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
+| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| repo-enum        | 1           | Enumerates code repositories                       | code_repository                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 <!-- END BBOT MODULE FLAGS -->
 
 ## BBOT Output Modules
diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md
index 019b4caaf..0a0cf21d5 100644
--- a/docs/modules/list_of_modules.md
+++ b/docs/modules/list_of_modules.md
@@ -58,6 +58,7 @@
 | censys               | scan     | Yes             | Query the Censys API                                                                                                                    | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | certspotter          | scan     | No              | Query Certspotter's API for subdomains                                                                                                  | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | chaos                | scan     | Yes             | Query ProjectDiscovery's Chaos API for subdomains                                                                                       | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
+| code_repository      | scan     | No              | Look for code repository links in webpages                                                                                              | passive, repo-enum, safe                                                       | URL_UNVERIFIED                                                                                                                          | CODE_REPOSITORY                                             |
 | columbus             | scan     | No              | Query the Columbus Project API for subdomains                                                                                           | passive, safe, subdomain-enum                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
 | credshed             | scan     | Yes             | Send queries to your own credshed server to check for known credentials of your targets                                                 | passive, safe                                                                  | DNS_NAME                                                                                                                                | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME          |
 | crobat               | scan     | No              | Query Project Crobat for subdomains                                                                                                     | passive, safe                                                                  | DNS_NAME                                                                                                                                | DNS_NAME                                                    |
diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md
index dbd9cbb37..6ba8997f8 100644
--- a/docs/scanning/advanced.md
+++ b/docs/scanning/advanced.md
@@ -61,12 +61,12 @@ Target:
 
 Modules:
   -m MODULE [MODULE ...], --modules MODULE [MODULE ...]
-                        Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,baddns,baddns_zone,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,docker_pull,dockerhub,dotnetnuke,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,github_codesearch,github_org,github_workflows,gitlab,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,newsletters,nmap,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,sublist3r,telerik,threatminer,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
+                        Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,baddns,baddns_zone,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,code_repository,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,docker_pull,dockerhub,dotnetnuke,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,git_clone,github_codesearch,github_org,github_workflows,gitlab,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,newsletters,nmap,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomaincenter,sublist3r,telerik,threatminer,trufflehog,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
   -l, --list-modules    List available modules.
   -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...]
                         Exclude these modules.
   -f FLAG [FLAG ...], --flags FLAG [FLAG ...]
-                        Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,deadly,email-enum,iis-shortnames,passive,portscan,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough
+                        Enable modules by flag. Choices: active,affiliates,aggressive,baddns,cloud-enum,deadly,email-enum,iis-shortnames,passive,portscan,repo-enum,report,safe,service-enum,slow,social-enum,subdomain-enum,subdomain-hijack,web-basic,web-paramminer,web-screenshots,web-thorough
   -lf, --list-flags     List available flags.
   -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...]
                         Only enable modules with these flags (e.g. -rf passive)
diff --git a/docs/scanning/events.md b/docs/scanning/events.md
index dad28f185..7c9d37b41 100644
--- a/docs/scanning/events.md
+++ b/docs/scanning/events.md
@@ -54,7 +54,7 @@ Below is a full list of event types along with which modules produce/consume the
 | *                   | 12                    | 0                     | affiliates, csv, discord, http, human, json, neo4j, python, slack, splunk, teams, websocket                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |                                                                                                                                                                                                                                                                                                                                                                                                                                            |
 | ASN                 | 0                     | 1                     |                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | asn                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | AZURE_TENANT        | 1                     | 0                     | speculate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| CODE_REPOSITORY     | 3                     | 4                     | docker_pull, git_clone, github_workflows                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | dockerhub, github_codesearch, github_org, gitlab                                                                                                                                                                                                                                                                                                                                                                                           |
+| CODE_REPOSITORY     | 3                     | 5                     | docker_pull, git_clone, github_workflows                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | code_repository, dockerhub, github_codesearch, github_org, gitlab                                                                                                                                                                                                                                                                                                                                                                          |
 | DNS_NAME            | 57                    | 42                    | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, massdns, myssl, nmap, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomains, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, massdns, myssl, ntlm, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, sublist3r, threatminer, urlscan, vhost, viewdns, virustotal, wayback, zoomeye |
 | DNS_NAME_UNRESOLVED | 3                     | 0                     | baddns, speculate, subdomains                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |                                                                                                                                                                                                                                                                                                                                                                                                                                            |
 | EMAIL_ADDRESS       | 1                     | 6                     | emails                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | credshed, emailformat, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                      |
@@ -74,7 +74,7 @@ Below is a full list of event types along with which modules produce/consume the
 | TECHNOLOGY          | 3                     | 7                     | asset_inventory, gitlab, web_report                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | badsecrets, dotnetnuke, gitlab, gowitness, internetdb, nuclei, wappalyzer                                                                                                                                                                                                                                                                                                                                                                  |
 | URL                 | 19                    | 2                     | ajaxpro, asset_inventory, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report                                                                                                                                                                                                                                                                                                                                                                                                                                                      | gowitness, httpx                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | URL_HINT            | 1                     | 1                     | ffuf_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       | iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| URL_UNVERIFIED      | 5                     | 14                    | filedownload, httpx, oauth, social, speculate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | azure_realm, bevigil, bucket_file_enum, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, postman, robots, urlscan, wayback                                                                                                                                                                                                                                                                              |
+| URL_UNVERIFIED      | 6                     | 14                    | code_repository, filedownload, httpx, oauth, social, speculate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | azure_realm, bevigil, bucket_file_enum, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, postman, robots, urlscan, wayback                                                                                                                                                                                                                                                                              |
 | USERNAME            | 1                     | 2                     | speculate                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | credshed, dehashed                                                                                                                                                                                                                                                                                                                                                                                                                         |
 | VHOST               | 1                     | 1                     | web_report                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | vhost                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | VULNERABILITY       | 2                     | 11                    | asset_inventory, web_report                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | ajaxpro, baddns, baddns_zone, badsecrets, dastardly, dotnetnuke, generic_ssrf, internetdb, nuclei, telerik, trufflehog                                                                                                                                                                                                                                                                                                                     |
diff --git a/docs/scanning/index.md b/docs/scanning/index.md
index 588c48b0e..21615a6fb 100644
--- a/docs/scanning/index.md
+++ b/docs/scanning/index.md
@@ -107,29 +107,30 @@ A single module can have multiple flags. For example, the `securitytrails` modul
 ### List of Flags
 
 <!-- BBOT MODULE FLAGS -->
-| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
-|------------------|-------------|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| safe             | 81          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
-| passive          | 61          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
-| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                              |
-| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
-| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
-| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
-| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
-| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
-| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
-| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
-| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |
+| Flag             | # Modules   | Description                                        | Modules                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+|------------------|-------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| safe             | 82          | Non-intrusive, safe to run                         | affiliates, aggregate, ajaxpro, anubisdb, asn, azure_realm, azure_tenant, baddns, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, postman, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye |
+| passive          | 62          | Never connects to target systems                   | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                                      |
+| subdomain-enum   | 45          | Enumerates subdomains                              | anubisdb, asn, azure_realm, azure_tenant, baddns_zone, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| active           | 42          | Makes active connections to target systems         | ajaxpro, baddns, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, newsletters, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| web-thorough     | 29          | More advanced web scanning functionality           | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dotnetnuke, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| aggressive       | 20          | Generates a large amount of network traffic        | bypass403, dastardly, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| web-basic        | 17          | Basic, non-intrusive web scan functionality        | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
+| cloud-enum       | 12          | Enumerates cloud resources                         | azure_realm, azure_tenant, baddns, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
+| slow             | 10          | May take a long time to complete                   | bucket_digitalocean, dastardly, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| affiliates       | 8           | Discovers affiliated hostnames/domains             | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
+| email-enum       | 7           | Enumerates email addresses                         | dehashed, emailformat, emails, hunterio, pgp, skymem, sslcert                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| deadly           | 4           | Highly aggressive                                  | dastardly, ffuf, nuclei, vhost                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
+| portscan         | 3           | Discovers open ports                               | internetdb, masscan, nmap                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| web-paramminer   | 3           | Discovers HTTP parameters through brute-force      | paramminer_cookies, paramminer_getparams, paramminer_headers                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| baddns           | 2           | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_zone                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
+| iis-shortnames   | 2           | Scans for IIS Shortname vulnerability              | ffuf_shortnames, iis_shortnames                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| report           | 2           | Generates a report at the end of the scan          | affiliates, asn                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| social-enum      | 2           | Enumerates social media                            | httpx, social                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+| repo-enum        | 1           | Enumerates code repositories                       | code_repository                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| service-enum     | 1           | Identifies protocols running on open ports         | fingerprintx                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| subdomain-hijack | 1           | Detects hijackable subdomains                      | baddns                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
+| web-screenshots  | 1           | Takes screenshots of web pages                     | gowitness                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 <!-- END BBOT MODULE FLAGS --> 
 
 ## Dependencies

From 464624186c246d91fce10c29c2d31808daff9994 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 15:30:45 -0400
Subject: [PATCH 145/220] send SIGINT on process timeout

---
 bbot/core/helpers/command.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index 785d646cb..c1c375caa 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -2,6 +2,7 @@
 import asyncio
 import logging
 import traceback
+from signal import SIGINT
 from subprocess import CompletedProcess, CalledProcessError
 
 from .misc import smart_decode, smart_encode
@@ -46,10 +47,15 @@ async def run(self, *command, check=False, text=True, idle_timeout=None, **kwarg
                     _input = b"\n".join(smart_encode(i) for i in _input) + b"\n"
                 else:
                     _input = smart_encode(_input)
-            if idle_timeout is not None:
-                stdout, stderr = await asyncio.wait_for(proc.communicate(_input), timeout=idle_timeout)
-            else:
-                stdout, stderr = await proc.communicate(_input)
+
+            try:
+                if idle_timeout is not None:
+                    stdout, stderr = await asyncio.wait_for(proc.communicate(_input), timeout=idle_timeout)
+                else:
+                    stdout, stderr = await proc.communicate(_input)
+            except TimeoutError:
+                proc.send_signal(SIGINT)
+                raise
 
             # surface stderr
             if text:
@@ -111,6 +117,9 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
                         line = await asyncio.wait_for(proc.stdout.readline(), timeout=idle_timeout)
                     else:
                         line = await proc.stdout.readline()
+                except TimeoutError as e:
+                    proc.send_signal(SIGINT)
+                    raise
                 except ValueError as e:
                     command_str = " ".join([str(c) for c in command])
                     log.warning(f"Error executing command {command_str}: {e}")

From 6f09df32abf2b5c00bae0ec4d1b8a9f5f63345a0 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 15:38:24 -0400
Subject: [PATCH 146/220] lower default masscan rate, add safe tag

---
 bbot/modules/portscan.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index a6fc18bfd..62fbfea75 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -7,7 +7,7 @@
 
 
 class portscan(BaseModule):
-    flags = ["active", "portscan"]
+    flags = ["active", "portscan", "safe"]
     watched_events = ["IP_ADDRESS", "IP_RANGE", "DNS_NAME"]
     produced_events = ["OPEN_TCP_PORT"]
     meta = {"description": "Port scan with masscan. By default, scans top 100 ports."}
@@ -15,7 +15,7 @@ class portscan(BaseModule):
         "top_ports": 100,
         "ports": "",
         # ping scan at 600 packets/s ~= private IP space in 8 hours
-        "rate": 600,
+        "rate": 300,
         "wait": 5,
         "ping_first": False,
         "ping_only": False,
@@ -33,7 +33,7 @@ class portscan(BaseModule):
 
     async def setup(self):
         self.top_ports = self.config.get("top_ports", 100)
-        self.rate = self.config.get("rate", 600)
+        self.rate = self.config.get("rate", 300)
         self.wait = self.config.get("wait", 10)
         self.ping_first = self.config.get("ping_first", False)
         self.ping_only = self.config.get("ping_only", False)

From 72070a26c2a3e0f33020b336b9119cbd31396d0c Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 15:46:35 -0400
Subject: [PATCH 147/220] fix github workflow tests

---
 .../test_module_github_workflows.py             | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
index c5bdf6d07..4cb6fff41 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_workflows.py
@@ -439,7 +439,7 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-        assert len(events) == 7
+        assert len(events) == 8
         assert 1 == len(
             [
                 e
@@ -457,7 +457,20 @@ def check(self, module_test, events):
                 if e.type == "SOCIAL"
                 and e.data["platform"] == "github"
                 and e.data["profile_name"] == "blacklanternsecurity"
-                and "github-org" in e.tags
+                and e.data["url"] == "https://github.com/blacklanternsecurity"
+                and str(e.module) == "github_org"
+                and e.scope_distance == 1
+            ]
+        ), "Failed to find blacklanternsecurity github"
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "SOCIAL"
+                and e.data["platform"] == "github"
+                and e.data["profile_name"] == "blacklanternsecurity"
+                and e.data["url"] == "https://github.com/blacklanternsecurity"
+                and str(e.module) == "social"
                 and e.scope_distance == 1
             ]
         ), "Failed to find blacklanternsecurity github"

From d18f9761f9d6cfa16f26e51e0a6bf273d58579c7 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 16:24:22 -0400
Subject: [PATCH 148/220] flaked

---
 bbot/core/helpers/command.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index c1c375caa..354ef7d4a 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -117,7 +117,7 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
                         line = await asyncio.wait_for(proc.stdout.readline(), timeout=idle_timeout)
                     else:
                         line = await proc.stdout.readline()
-                except TimeoutError as e:
+                except TimeoutError:
                     proc.send_signal(SIGINT)
                     raise
                 except ValueError as e:

From 13e8a4fb26b97dd336553096ab87b289b08c62bd Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 16:44:41 -0400
Subject: [PATCH 149/220] update release history

---
 docs/release_history.md | 59 ++++++++++++++++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 3 deletions(-)

diff --git a/docs/release_history.md b/docs/release_history.md
index 4b5b15bac..9de5ef6ca 100644
--- a/docs/release_history.md
+++ b/docs/release_history.md
@@ -1,7 +1,60 @@
+## v1.1.7
+May 15th, 2024
+
+### New Modules
+- https://github.com/blacklanternsecurity/bbot/pull/1037
+- https://github.com/blacklanternsecurity/bbot/pull/1122
+- https://github.com/blacklanternsecurity/bbot/pull/1176
+- https://github.com/blacklanternsecurity/bbot/pull/1164
+- https://github.com/blacklanternsecurity/bbot/pull/1169
+- https://github.com/blacklanternsecurity/bbot/pull/1175
+- https://github.com/blacklanternsecurity/bbot/pull/1209
+- https://github.com/blacklanternsecurity/bbot/pull/1335
+- https://github.com/blacklanternsecurity/bbot/pull/1380
+
+### Improvements
+- https://github.com/blacklanternsecurity/bbot/pull/1132
+- https://github.com/blacklanternsecurity/bbot/pull/1156
+- https://github.com/blacklanternsecurity/bbot/pull/1160
+- https://github.com/blacklanternsecurity/bbot/pull/1162
+- https://github.com/blacklanternsecurity/bbot/pull/1165
+- https://github.com/blacklanternsecurity/bbot/pull/1179
+- https://github.com/blacklanternsecurity/bbot/pull/1182
+- https://github.com/blacklanternsecurity/bbot/pull/1185
+- https://github.com/blacklanternsecurity/bbot/pull/1186
+- https://github.com/blacklanternsecurity/bbot/pull/1197
+- https://github.com/blacklanternsecurity/bbot/pull/1198
+- https://github.com/blacklanternsecurity/bbot/pull/1205
+- https://github.com/blacklanternsecurity/bbot/pull/1217
+- https://github.com/blacklanternsecurity/bbot/pull/1233
+- https://github.com/blacklanternsecurity/bbot/pull/1283
+- https://github.com/blacklanternsecurity/bbot/pull/1288
+- https://github.com/blacklanternsecurity/bbot/pull/1296
+- https://github.com/blacklanternsecurity/bbot/pull/1306
+- https://github.com/blacklanternsecurity/bbot/pull/1313
+- https://github.com/blacklanternsecurity/bbot/pull/1349
+- https://github.com/blacklanternsecurity/bbot/pull/1343
+
+### Bugfixes
+- https://github.com/blacklanternsecurity/bbot/pull/1136
+- https://github.com/blacklanternsecurity/bbot/pull/1140
+- https://github.com/blacklanternsecurity/bbot/pull/1152
+- https://github.com/blacklanternsecurity/bbot/pull/1154
+- https://github.com/blacklanternsecurity/bbot/pull/1181
+- https://github.com/blacklanternsecurity/bbot/pull/1247
+- https://github.com/blacklanternsecurity/bbot/pull/1249
+- https://github.com/blacklanternsecurity/bbot/pull/1273
+- https://github.com/blacklanternsecurity/bbot/pull/1277
+- https://github.com/blacklanternsecurity/bbot/pull/1287
+- https://github.com/blacklanternsecurity/bbot/pull/1298
+- https://github.com/blacklanternsecurity/bbot/pull/1308
+- https://github.com/blacklanternsecurity/bbot/pull/1336
+- https://github.com/blacklanternsecurity/bbot/pull/1348
+
 ## v1.1.6
 February 21, 2024
 
-## Improvements
+### Improvements
 - https://github.com/blacklanternsecurity/bbot/pull/1001
 - https://github.com/blacklanternsecurity/bbot/pull/1006
 - https://github.com/blacklanternsecurity/bbot/pull/1010
@@ -22,7 +75,7 @@ February 21, 2024
 - https://github.com/blacklanternsecurity/bbot/pull/1101
 - https://github.com/blacklanternsecurity/bbot/pull/1103
 
-## Bigfixes
+### Bigfixes
 - https://github.com/blacklanternsecurity/bbot/pull/1005
 - https://github.com/blacklanternsecurity/bbot/pull/1022
 - https://github.com/blacklanternsecurity/bbot/pull/1030
@@ -37,7 +90,7 @@ February 21, 2024
 - https://github.com/blacklanternsecurity/bbot/pull/1094
 - https://github.com/blacklanternsecurity/bbot/pull/1098
 
-## New Modules
+### New Modules
 - https://github.com/blacklanternsecurity/bbot/pull/1072
 - https://github.com/blacklanternsecurity/bbot/pull/1091
 

From 5c174fb9d1a222350d4d856969bd45717b123b63 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 16:48:23 -0400
Subject: [PATCH 150/220] fix asset inventory tests

---
 .../module_tests/test_module_asset_inventory.py | 17 +++++++++++++++++
 .../module_tests/test_module_portscan.py        | 10 ++++++----
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
index c9703ea92..615113504 100644
--- a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
+++ b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
@@ -7,7 +7,24 @@ class TestAsset_Inventory(ModuleTestBase):
     config_overrides = {"dns_resolution": True, "modules": {"portscan": {"ports": "9999"}}}
     modules_overrides = ["asset_inventory", "portscan", "sslcert"]
 
+    masscan_output = """{   "ip": "127.0.0.1",   "timestamp": "1680197558", "ports": [ {"port": 9999, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }"""
+
     async def setup_before_prep(self, module_test):
+
+        async def run_masscan(command, *args, **kwargs):
+            if "masscan" in command[:2]:
+                targets = open(command[11]).read().splitlines()
+                yield "["
+                for l in self.masscan_output.splitlines():
+                    if "127.0.0.1/32" in targets:
+                        yield self.masscan_output
+                yield "]"
+            else:
+                async for l in module_test.scan.helpers.run_live(command, *args, **kwargs):
+                    yield l
+
+        module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", run_masscan)
+
         await module_test.mock_dns(
             {
                 "1.0.0.127.in-addr.arpa": {"PTR": ["www.bbottest.notreal"]},
diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py
index cae6cb20c..93c55f715 100644
--- a/bbot/test/test_step_2/module_tests/test_module_portscan.py
+++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py
@@ -6,17 +6,19 @@ class TestPortscan(ModuleTestBase):
     scan_name = "test_portscan"
     config_overrides = {"modules": {"portscan": {"ports": "443", "wait": 1}}}
 
-    masscan_output = """[
-{   "ip": "8.8.8.8",   "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }
-]"""
+    masscan_output = """{   "ip": "8.8.8.8",   "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }"""
 
     async def setup_after_prep(self, module_test):
         self.masscan_run = False
 
         async def run_masscan(command, *args, **kwargs):
             if "masscan" in command[:2]:
+                targets = open(command[11]).read().splitlines()
+                yield "["
                 for l in self.masscan_output.splitlines():
-                    yield l
+                    if "8.8.8.8/32" in targets:
+                        yield self.masscan_output
+                yield "]"
                 self.masscan_run = True
             else:
                 async for l in module_test.scan.helpers.run_live(command, *args, **kwargs):

From 6b57a56240ca72b3d5deb5be9c1373a559fd03b2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 16:56:36 -0400
Subject: [PATCH 151/220] updated exception class

---
 bbot/core/helpers/command.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index 354ef7d4a..59751cbee 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -53,7 +53,7 @@ async def run(self, *command, check=False, text=True, idle_timeout=None, **kwarg
                     stdout, stderr = await asyncio.wait_for(proc.communicate(_input), timeout=idle_timeout)
                 else:
                     stdout, stderr = await proc.communicate(_input)
-            except TimeoutError:
+            except asyncio.exceptions.TimeoutError:
                 proc.send_signal(SIGINT)
                 raise
 
@@ -117,7 +117,7 @@ async def run_live(self, *command, check=False, text=True, idle_timeout=None, **
                         line = await asyncio.wait_for(proc.stdout.readline(), timeout=idle_timeout)
                     else:
                         line = await proc.stdout.readline()
-                except TimeoutError:
+                except asyncio.exceptions.TimeoutError:
                     proc.send_signal(SIGINT)
                     raise
                 except ValueError as e:

From a35f10c87c758cf38ee1fe68cbe5c53a0861cc4b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 16:58:05 -0400
Subject: [PATCH 152/220] updated classes in tests

---
 bbot/test/test_step_1/test_command.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py
index 0651f857a..42e41e4da 100644
--- a/bbot/test/test_step_1/test_command.py
+++ b/bbot/test/test_step_1/test_command.py
@@ -10,14 +10,14 @@ async def test_command(bbot_scanner, bbot_config):
     # test timeouts
     command = ["sleep", "3"]
     start = time.time()
-    with pytest.raises(TimeoutError):
+    with pytest.raises(asyncio.exceptions.TimeoutError):
         await scan1.helpers.run(command, idle_timeout=1)
     end = time.time()
     elapsed = end - start
     assert 0 < elapsed < 2
 
     start = time.time()
-    with pytest.raises(TimeoutError):
+    with pytest.raises(asyncio.exceptions.TimeoutError):
         async for line in scan1.helpers.run_live(command, idle_timeout=1):
             print(line)
     end = time.time()

From 63ddd6e90087caf273ec1143808070ebe7c4947a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 15 May 2024 17:02:42 -0400
Subject: [PATCH 153/220] fix gowitness

---
 bbot/modules/gowitness.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 2b7f9291e..ca308202a 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -1,3 +1,4 @@
+import asyncio
 import sqlite3
 import multiprocessing
 from pathlib import Path
@@ -164,7 +165,7 @@ async def handle_batch(self, *events):
         try:
             async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):
                 self.debug(line)
-        except TimeoutError:
+        except asyncio.exceptions.TimeoutError:
             urls_str = ",".join(event_dict)
             self.warning(f"Gowitness timed out while visiting the following URLs: {urls_str}", trace=False)
             return

From 7c91de6c1de5921ab5ffa40151e0e0bdf828ee01 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 09:38:49 -0400
Subject: [PATCH 154/220] rename event.source to event.parent

---
 bbot/core/event/base.py                       | 163 +++++++++---------
 bbot/core/helpers/web.py                      |  10 +-
 bbot/modules/azure_realm.py                   |   2 +-
 bbot/modules/azure_tenant.py                  |   4 +-
 bbot/modules/baddns_zone.py                   |   2 +-
 bbot/modules/base.py                          |   6 +-
 bbot/modules/bevigil.py                       |   4 +-
 bbot/modules/bucket_file_enum.py              |   2 +-
 bbot/modules/builtwith.py                     |   4 +-
 bbot/modules/bypass403.py                     |   4 +-
 bbot/modules/code_repository.py               |   2 +-
 bbot/modules/credshed.py                      |   8 +-
 bbot/modules/deadly/ffuf.py                   |   2 +-
 bbot/modules/deadly/nuclei.py                 |  20 +--
 bbot/modules/deadly/vhost.py                  |   4 +-
 bbot/modules/dehashed.py                      |   8 +-
 bbot/modules/dnsbrute.py                      |   2 +-
 bbot/modules/dnsbrute_mutations.py            |  20 +--
 bbot/modules/dnscommonsrv.py                  |   2 +-
 bbot/modules/docker_pull.py                   |   2 +-
 bbot/modules/dockerhub.py                     |   6 +-
 bbot/modules/emailformat.py                   |   2 +-
 bbot/modules/ffuf_shortnames.py               |  22 +--
 bbot/modules/fingerprintx.py                  |   4 +-
 bbot/modules/git_clone.py                     |   2 +-
 bbot/modules/github_codesearch.py             |   4 +-
 bbot/modules/github_org.py                    |   6 +-
 bbot/modules/github_workflows.py              |   2 +-
 bbot/modules/gitlab.py                        |   8 +-
 bbot/modules/gowitness.py                     |  22 +--
 bbot/modules/httpx.py                         |  14 +-
 bbot/modules/internal/cloud.py                |   2 +-
 bbot/modules/internal/dns.py                  |  16 +-
 bbot/modules/internal/excavate.py             |  18 +-
 bbot/modules/internal/speculate.py            |  24 +--
 bbot/modules/internetdb.py                    |   8 +-
 bbot/modules/masscan.py                       |  54 +++---
 bbot/modules/nmap.py                          |  16 +-
 bbot/modules/ntlm.py                          |   4 +-
 bbot/modules/oauth.py                         |   8 +-
 bbot/modules/output/asset_inventory.py        |   8 +-
 bbot/modules/output/neo4j.py                  |   2 +-
 bbot/modules/output/web_report.py             |  22 +--
 bbot/modules/postman.py                       |   2 +-
 bbot/modules/report/asn.py                    |   4 +-
 bbot/modules/robots.py                        |   2 +-
 bbot/modules/secretsdb.py                     |   2 +-
 bbot/modules/skymem.py                        |   4 +-
 bbot/modules/smuggler.py                      |   2 +-
 bbot/modules/social.py                        |   2 +-
 bbot/modules/sslcert.py                       |  12 +-
 bbot/modules/templates/bucket.py              |   6 +-
 bbot/modules/trufflehog.py                    |   4 +-
 bbot/modules/url_manipulation.py              |   2 +-
 bbot/modules/urlscan.py                       |  10 +-
 bbot/modules/wafw00f.py                       |   4 +-
 bbot/scanner/manager.py                       |  32 ++--
 bbot/scanner/scanner.py                       |   4 +-
 bbot/test/bbot_fixtures.py                    |  50 +++---
 bbot/test/test_step_1/test_dns.py             |  16 +-
 bbot/test/test_step_1/test_events.py          | 128 +++++++-------
 bbot/test/test_step_1/test_helpers.py         |   2 +-
 .../test_step_1/test_manager_deduplication.py |  84 ++++-----
 .../test_manager_scope_accuracy.py            |   6 +-
 bbot/test/test_step_1/test_modules_basic.py   |  28 +--
 bbot/test/test_step_1/test_scan.py            |   6 +-
 .../module_tests/test_module_cloud.py         |  22 +--
 .../module_tests/test_module_dehashed.py      |   2 +-
 .../module_tests/test_module_dnsbrute.py      |  12 +-
 .../module_tests/test_module_github_org.py    |   4 +-
 .../module_tests/test_module_gowitness.py     |   2 +-
 .../module_tests/test_module_nmap.py          |  14 +-
 .../module_tests/test_module_trufflehog.py    |  16 +-
 docs/dev/module_howto.md                      |   2 +-
 docs/scanning/events.md                       |   4 +-
 docs/scanning/output.md                       |   2 +-
 76 files changed, 520 insertions(+), 517 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 66116d87d..98314095a 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1,11 +1,11 @@
 import re
 import json
 import logging
+import datetime
 import ipaddress
 import traceback
 from copy import copy
 from typing import Optional
-from datetime import datetime
 from contextlib import suppress
 from urllib.parse import urljoin
 from radixtarget import RadixTarget
@@ -65,8 +65,8 @@ class BaseEvent:
         scan (Scanner): The scan object that generated the event.
         timestamp (datetime.datetime): The time at which the data was discovered.
         resolved_hosts (list of str): List of hosts to which the event data resolves, applicable for URLs and DNS names.
-        source (BaseEvent): The source event that led to the discovery of this event.
-        source_id (str): The `id` attribute of the source event.
+        parent (BaseEvent): The parent event that led to the discovery of this event.
+        parent_id (str): The `id` attribute of the parent event.
         tags (set of str): Descriptive tags for the event, e.g., `mx-record`, `in-scope`.
         module (BaseModule): The module that discovered the event.
         module_sequence (str): The sequence of modules that participated in the discovery.
@@ -82,7 +82,7 @@ class BaseEvent:
             "scan": "SCAN:4d786912dbc97be199da13074699c318e2067a7f",
             "timestamp": 1688526222.723366,
             "resolved_hosts": ["185.199.108.153"],
-            "source": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7",
+            "parent": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7",
             "tags": ["in-scope", "distance-0", "dir", "ip-185-199-108-153", "status-301", "http-title-301-moved-permanently"],
             "module": "httpx",
             "module_sequence": "httpx"
@@ -110,7 +110,7 @@ def __init__(
         self,
         data,
         event_type,
-        source=None,
+        parent=None,
         module=None,
         scan=None,
         scans=None,
@@ -129,7 +129,7 @@ def __init__(
         Attributes:
             data (str, dict): The primary data for the event.
             event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'.
-            source (BaseEvent, optional): Source event that led to this event's discovery. Defaults to None.
+            parent (BaseEvent, optional): Parent event that led to this event's discovery. Defaults to None.
             module (str, optional): Module that discovered the event. Defaults to None.
             scan (Scan, optional): BBOT Scan object. Required unless _dummy is True. Defaults to None.
             scans (list of Scan, optional): BBOT Scan objects, used primarily when unserializing an Event from the database. Defaults to None.
@@ -140,7 +140,7 @@ def __init__(
             _internal (Any, optional): If specified, makes the event internal. Defaults to None.
 
         Raises:
-            ValidationError: If either `scan` or `source` are not specified and `_dummy` is False.
+            ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.
         """
 
         self._id = None
@@ -158,7 +158,10 @@ def __init__(
         # keep track of whether this event has been recorded by the scan
         self._stats_recorded = False
 
-        self.timestamp = datetime.utcnow()
+        if timestamp is not None:
+            self.timestamp = timestamp
+        else:
+            self.timestamp = datetime.datetime.now(datetime.UTC)
 
         self._tags = set()
         if tags is not None:
@@ -168,7 +171,7 @@ def __init__(
         self._type = event_type
         self.confidence = int(confidence)
 
-        # for creating one-off events without enforcing source requirement
+        # for creating one-off events without enforcing parent requirement
         self._dummy = _dummy
         self._internal = False
 
@@ -195,20 +198,20 @@ def __init__(
         if not self.data:
             raise ValidationError(f'Invalid event data "{data}" for type "{self.type}"')
 
-        self._source = None
-        self._source_id = None
-        self.source = source
-        if (not self.source) and (not self._dummy):
-            raise ValidationError(f"Must specify event source")
+        self._parent = None
+        self._parent_id = None
+        self.parent = parent
+        if (not self.parent) and (not self._dummy):
+            raise ValidationError(f"Must specify event parent")
 
         # internal events are not ingested by output modules
         if not self._dummy:
             # removed this second part because it was making certain sslcert events internal
-            if _internal:  # or source._internal:
+            if _internal:  # or parent._internal:
                 self.internal = True
 
         # inherit web spider distance from parent
-        self.web_spider_distance = getattr(self.source, "web_spider_distance", 0)
+        self.web_spider_distance = getattr(self.parent, "web_spider_distance", 0)
 
     @property
     def data(self):
@@ -398,9 +401,9 @@ def scope_distance(self, scope_distance):
                     self.add_tag(f"distance-{new_scope_distance}")
                 self._scope_distance = new_scope_distance
             # apply recursively to parent events
-            source_scope_distance = getattr(self.source, "scope_distance", -1)
-            if source_scope_distance >= 0 and self != self.source:
-                self.source.scope_distance = scope_distance + 1
+            parent_scope_distance = getattr(self.parent, "scope_distance", -1)
+            if parent_scope_distance >= 0 and self != self.parent:
+                self.parent.scope_distance = scope_distance + 1
 
     @property
     def scope_description(self):
@@ -416,71 +419,71 @@ def scope_description(self):
         return f"distance-{self.scope_distance}"
 
     @property
-    def source(self):
-        return self._source
+    def parent(self):
+        return self._parent
 
-    @source.setter
-    def source(self, source):
+    @parent.setter
+    def parent(self, parent):
         """
-        Setter for the source attribute, ensuring it's a valid event and updating scope distance.
+        Setter for the parent attribute, ensuring it's a valid event and updating scope distance.
 
-        Sets the source of the event and automatically adjusts the scope distance based on the source event's
-        scope distance. The scope distance is incremented by 1 if the host of the source event is different
+        Sets the parent of the event and automatically adjusts the scope distance based on the parent event's
+        scope distance. The scope distance is incremented by 1 if the host of the parent event is different
         from the current event's host.
 
         Parameters:
-            source (BaseEvent): The new source event to set. Must be a valid event object.
+            parent (BaseEvent): The new parent event to set. Must be a valid event object.
 
         Note:
-            If an invalid source is provided and the event is not a dummy, a warning will be logged.
+            If an invalid parent is provided and the event is not a dummy, a warning will be logged.
         """
-        if is_event(source):
-            self._source = source
-            hosts_are_same = self.host and (self.host == source.host)
-            if source.scope_distance >= 0:
-                new_scope_distance = int(source.scope_distance)
+        if is_event(parent):
+            self._parent = parent
+            hosts_are_same = self.host and (self.host == parent.host)
+            if parent.scope_distance >= 0:
+                new_scope_distance = int(parent.scope_distance)
                 # only increment the scope distance if the host changes
                 if self._scope_distance_increment_same_host or not hosts_are_same:
                     new_scope_distance += 1
                 self.scope_distance = new_scope_distance
             # inherit certain tags
             if hosts_are_same:
-                for t in source.tags:
+                for t in parent.tags:
                     if t == "affiliate":
                         self.add_tag("affiliate")
                     elif t.startswith("mutation-"):
                         self.add_tag(t)
         elif not self._dummy:
-            log.warning(f"Tried to set invalid source on {self}: (got: {source})")
+            log.warning(f"Tried to set invalid parent on {self}: (got: {parent})")
 
     @property
-    def source_id(self):
-        source_id = getattr(self.get_source(), "id", None)
-        if source_id is not None:
-            return source_id
-        return self._source_id
+    def parent_id(self):
+        parent_id = getattr(self.get_parent(), "id", None)
+        if parent_id is not None:
+            return parent_id
+        return self._parent_id
 
-    def get_source(self):
+    def get_parent(self):
         """
         Takes into account events with the _omit flag
         """
-        if getattr(self.source, "_omit", False):
-            return self.source.get_source()
-        return self.source
+        if getattr(self.parent, "_omit", False):
+            return self.parent.get_parent()
+        return self.parent
 
-    def get_sources(self, omit=False):
-        sources = []
+    def get_parents(self, omit=False):
+        parents = []
         e = self
         while 1:
             if omit:
-                source = e.get_source()
+                parent = e.get_parent()
             else:
-                source = e.source
-            if e == source:
+                parent = e.parent
+            if e == parent:
                 break
-            sources.append(source)
-            e = source
-        return sources
+            parents.append(parent)
+            e = parent
+        return parents
 
     def _host(self):
         return ""
@@ -641,13 +644,13 @@ def json(self, mode="json", siem_friendly=False):
         # timestamp
         j["timestamp"] = self.timestamp.timestamp()
         # parent event
-        source_id = self.source_id
-        if source_id:
-            j["source"] = source_id
+        parent_id = self.parent_id
+        if parent_id:
+            j["parent"] = parent_id
         # tags
         if self.tags:
             j.update({"tags": list(self.tags)})
-        # source module
+        # parent module
         if self.module:
             j.update({"module": str(self.module)})
         # sequence of modules that led to discovery
@@ -685,14 +688,14 @@ def module_sequence(self):
         """
         Get a human-friendly string that represents the sequence of modules responsible for generating this event.
 
-        Includes the names of omitted source events to provide a complete view of the module sequence leading to this event.
+        Includes the names of omitted parent events to provide a complete view of the module sequence leading to this event.
 
         Returns:
             str: The module sequence in human-friendly format.
         """
         module_name = getattr(self.module, "name", "")
-        if getattr(self.source, "_omit", False):
-            module_name = f"{self.source.module_sequence}->{module_name}"
+        if getattr(self.parent, "_omit", False):
+            module_name = f"{self.parent.module_sequence}->{module_name}"
         return module_name
 
     @property
@@ -710,10 +713,10 @@ def module_priority(self, priority):
     def priority(self):
         if self._priority is None:
             timestamp = self.timestamp.timestamp()
-            if self.source.timestamp == self.timestamp:
+            if self.parent.timestamp == self.timestamp:
                 self._priority = (timestamp,)
             else:
-                self._priority = getattr(self.source, "priority", ()) + (timestamp,)
+                self._priority = getattr(self.parent, "priority", ()) + (timestamp,)
 
         return self._priority
 
@@ -827,7 +830,7 @@ def __init__(self, *args, **kwargs):
         self.add_tag(f"ipv{ip.version}")
         if ip.is_private:
             self.add_tag("private-ip")
-        self.dns_resolve_distance = getattr(self.source, "dns_resolve_distance", 0)
+        self.dns_resolve_distance = getattr(self.parent, "dns_resolve_distance", 0)
 
     def sanitize_data(self, data):
         return validators.validate_host(data)
@@ -841,14 +844,14 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         # prevent runaway DNS entries
         self.dns_resolve_distance = 0
-        source = getattr(self, "source", None)
+        parent = getattr(self, "parent", None)
         module = getattr(self, "module", None)
         module_type = getattr(module, "_type", "")
-        source_module = getattr(source, "module", None)
-        source_module_type = getattr(source_module, "_type", "")
+        parent_module = getattr(parent, "module", None)
+        parent_module_type = getattr(parent_module, "_type", "")
         if module_type == "DNS":
-            self.dns_resolve_distance = getattr(source, "dns_resolve_distance", 0)
-            if source_module_type == "DNS":
+            self.dns_resolve_distance = getattr(parent, "dns_resolve_distance", 0)
+            if parent_module_type == "DNS":
                 self.dns_resolve_distance += 1
         # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
 
@@ -915,7 +918,7 @@ def __init__(self, *args, **kwargs):
         # increment the web spider distance
         if self.type == "URL_UNVERIFIED":
             self.web_spider_distance += 1
-        self.num_redirects = getattr(self.source, "num_redirects", 0)
+        self.num_redirects = getattr(self.parent, "num_redirects", 0)
 
     def sanitize_data(self, data):
         self.parsed_url = validators.validate_url_parsed(data)
@@ -1024,7 +1027,7 @@ class HTTP_RESPONSE(URL_UNVERIFIED, DictEvent):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         # count number of consecutive redirects
-        self.num_redirects = getattr(self.source, "num_redirects", 0)
+        self.num_redirects = getattr(self.parent, "num_redirects", 0)
         if str(self.http_status).startswith("3"):
             self.num_redirects += 1
 
@@ -1232,7 +1235,7 @@ def _pretty_string(self):
 def make_event(
     data,
     event_type=None,
-    source=None,
+    parent=None,
     module=None,
     scan=None,
     scans=None,
@@ -1251,7 +1254,7 @@ def make_event(
     Parameters:
         data (Union[str, dict, BaseEvent]): The primary data for the event or an existing event object.
         event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'. Auto-detected if not provided.
-        source (BaseEvent, optional): Source event leading to this event's discovery.
+        parent (BaseEvent, optional): Parent event leading to this event's discovery.
         module (str, optional): Module that discovered the event.
         scan (Scan, optional): BBOT Scan object associated with the event.
         scans (List[Scan], optional): Multiple BBOT Scan objects, primarily used for unserialization.
@@ -1268,11 +1271,11 @@ def make_event(
 
     Examples:
         If inside a module, e.g. from within its `handle_event()`:
-        >>> self.make_event("1.2.3.4", source=event)
+        >>> self.make_event("1.2.3.4", parent=event)
         IP_ADDRESS("1.2.3.4", module=nmap, tags={'ipv4', 'distance-1'})
 
         If you're outside a module but you have a scan object:
-        >>> scan.make_event("1.2.3.4", source=scan.root_event)
+        >>> scan.make_event("1.2.3.4", parent=scan.root_event)
         IP_ADDRESS("1.2.3.4", module=None, tags={'ipv4', 'distance-1'})
 
         If you're outside a scan and just messing around:
@@ -1300,8 +1303,8 @@ def make_event(
             data.scans = scans
         if module is not None:
             data.module = module
-        if source is not None:
-            data.source = source
+        if parent is not None:
+            data.parent = parent
         if internal == True:
             data.internal = True
         if tags:
@@ -1343,7 +1346,7 @@ def make_event(
         return event_class(
             data,
             event_type=event_type,
-            source=source,
+            parent=parent,
             module=module,
             scan=scan,
             scans=scans,
@@ -1395,11 +1398,11 @@ def event_from_json(j, siem_friendly=False):
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
 
-        event.timestamp = datetime.fromtimestamp(j["timestamp"])
+        event.timestamp = datetime.datetime.fromtimestamp(j["timestamp"])
         event.scope_distance = j["scope_distance"]
-        source_id = j.get("source", None)
-        if source_id is not None:
-            event._source_id = source_id
+        parent_id = j.get("parent", None)
+        if parent_id is not None:
+            event._parent_id = parent_id
         return event
     except KeyError as e:
         raise ValidationError(f"Event missing required field: {e}")
diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py
index cbc049fc1..d58084b33 100644
--- a/bbot/core/helpers/web.py
+++ b/bbot/core/helpers/web.py
@@ -538,7 +538,7 @@ async def curl(self, *args, **kwargs):
         output = (await self.parent_helper.run(curl_command)).stdout
         return output
 
-    def is_spider_danger(self, source_event, url):
+    def is_spider_danger(self, parent_event, url):
         """
         Determines whether visiting a URL could potentially trigger a web-spider-like happening.
 
@@ -547,7 +547,7 @@ def is_spider_danger(self, source_event, url):
         the function returns True, indicating a possible web-spider risk.
 
         Args:
-            source_event: The source event object that discovered the URL.
+            parent_event: The parent event object that discovered the URL.
             url (str): The URL to evaluate for web-spider risk.
 
         Returns:
@@ -557,15 +557,15 @@ def is_spider_danger(self, source_event, url):
             - Write tests for this function
 
         Examples:
-            >>> is_spider_danger(source_event_obj, "https://example.com/subpage")
+            >>> is_spider_danger(parent_event_obj, "https://example.com/subpage")
             True
 
-            >>> is_spider_danger(source_event_obj, "https://example.com/")
+            >>> is_spider_danger(parent_event_obj, "https://example.com/")
             False
         """
         url_depth = self.parent_helper.url_depth(url)
         web_spider_depth = self.parent_helper.config.get("web_spider_depth", 1)
-        spider_distance = getattr(source_event, "web_spider_distance", 0) + 1
+        spider_distance = getattr(parent_event, "web_spider_distance", 0) + 1
         web_spider_distance = self.parent_helper.config.get("web_spider_distance", 0)
         if (url_depth > web_spider_depth) or (spider_distance > web_spider_distance):
             return True
diff --git a/bbot/modules/azure_realm.py b/bbot/modules/azure_realm.py
index a3d6ad6ba..dfc32a64c 100644
--- a/bbot/modules/azure_realm.py
+++ b/bbot/modules/azure_realm.py
@@ -19,7 +19,7 @@ async def handle_event(self, event):
             auth_url = await self.getuserrealm(domain)
             if auth_url:
                 url_event = self.make_event(
-                    auth_url, "URL_UNVERIFIED", source=event, tags=["affiliate", "ms-auth-url"]
+                    auth_url, "URL_UNVERIFIED", parent=event, tags=["affiliate", "ms-auth-url"]
                 )
                 url_event.source_domain = domain
                 await self.emit_event(url_event)
diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index a15bbb68f..3b12f4325 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -34,7 +34,7 @@ async def handle_event(self, event):
             self.verbose(f'Found {len(domains):,} domains under tenant for "{query}": {", ".join(sorted(domains))}')
             for domain in domains:
                 if domain != query:
-                    await self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate", "azure-tenant"])
+                    await self.emit_event(domain, "DNS_NAME", parent=event, tags=["affiliate", "azure-tenant"])
                     # tenant names
                     if domain.lower().endswith(".onmicrosoft.com"):
                         tenantname = domain.split(".")[0].lower()
@@ -44,7 +44,7 @@ async def handle_event(self, event):
             event_data = {"tenant-names": sorted(tenant_names), "domains": sorted(domains)}
             if tenant_id is not None:
                 event_data["tenant-id"] = tenant_id
-            await self.emit_event(event_data, "AZURE_TENANT", source=event)
+            await self.emit_event(event_data, "AZURE_TENANT", parent=event)
 
     async def query(self, domain):
         url = f"{self.base_url}/autodiscover/autodiscover.svc"
diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py
index ac0fc3c57..9501a25e6 100644
--- a/bbot/modules/baddns_zone.py
+++ b/bbot/modules/baddns_zone.py
@@ -24,6 +24,6 @@ def select_modules(self):
 
     # minimize nsec records feeding back into themselves
     async def filter_event(self, event):
-        if "baddns-nsec" in event.tags or "baddns-nsec" in event.source.tags:
+        if "baddns-nsec" in event.tags or "baddns-nsec" in event.parent.tags:
             return False
         return True
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 65de3299e..e17055d2e 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -415,7 +415,7 @@ def make_event(self, *args, **kwargs):
             raise_error (bool, optional): Whether to raise a validation error if the event could not be created. Defaults to False.
 
         Examples:
-            >>> new_event = self.make_event("1.2.3.4", source=event)
+            >>> new_event = self.make_event("1.2.3.4", parent=event)
             >>> await self.emit_event(new_event)
 
         Returns:
@@ -454,9 +454,9 @@ async def emit_event(self, *args, **kwargs):
                 ```
 
         Examples:
-            >>> await self.emit_event("www.evilcorp.com", source=event, tags=["affiliate"])
+            >>> await self.emit_event("www.evilcorp.com", parent=event, tags=["affiliate"])
 
-            >>> new_event = self.make_event("1.2.3.4", source=event)
+            >>> new_event = self.make_event("1.2.3.4", parent=event)
             >>> await self.emit_event(new_event)
 
         Returns:
diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py
index bbf339b08..91d1dcda1 100644
--- a/bbot/modules/bevigil.py
+++ b/bbot/modules/bevigil.py
@@ -29,13 +29,13 @@ async def handle_event(self, event):
         subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains)
         if subdomains:
             for subdomain in subdomains:
-                await self.emit_event(subdomain, "DNS_NAME", source=event)
+                await self.emit_event(subdomain, "DNS_NAME", parent=event)
 
         if self.urls:
             urls = await self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls)
             if urls:
                 for parsed_url in await self.helpers.run_in_executor_mp(self.helpers.validators.collapse_urls, urls):
-                    await self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", source=event)
+                    await self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", parent=event)
 
     async def request_subdomains(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/"
diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py
index facaa021e..c0f1ec179 100644
--- a/bbot/modules/bucket_file_enum.py
+++ b/bbot/modules/bucket_file_enum.py
@@ -42,7 +42,7 @@ async def handle_aws(self, event):
                 bucket_file = url + "/" + key
                 file_extension = self.helpers.get_file_extension(key)
                 if file_extension not in self.scan.url_extension_blacklist:
-                    await self.emit_event(bucket_file, "URL_UNVERIFIED", source=event, tags="filedownload")
+                    await self.emit_event(bucket_file, "URL_UNVERIFIED", parent=event, tags="filedownload")
                     urls_emitted += 1
                     if urls_emitted >= self.file_limit:
                         return
diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py
index 0b5793657..2bf60120e 100644
--- a/bbot/modules/builtwith.py
+++ b/bbot/modules/builtwith.py
@@ -33,14 +33,14 @@ async def handle_event(self, event):
         if subdomains:
             for s in subdomains:
                 if s != event:
-                    await self.emit_event(s, "DNS_NAME", source=event)
+                    await self.emit_event(s, "DNS_NAME", parent=event)
         # redirects
         if self.config.get("redirects", True):
             redirects = await self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects)
             if redirects:
                 for r in redirects:
                     if r != event:
-                        await self.emit_event(r, "DNS_NAME", source=event, tags=["affiliate"])
+                        await self.emit_event(r, "DNS_NAME", parent=event, tags=["affiliate"])
 
     async def request_domains(self, query):
         url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes"
diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index 95f3ef551..641ee9d6b 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -146,14 +146,14 @@ async def handle_event(self, event):
                     "url": event.data,
                 },
                 "FINDING",
-                source=event,
+                parent=event,
             )
         else:
             for description in results:
                 await self.emit_event(
                     {"description": description, "host": str(event.host), "url": event.data},
                     "FINDING",
-                    source=event,
+                    parent=event,
                 )
 
     # When a WAF-check helper is available in the future, we will convert to HTTP_RESPONSE and check for the WAF string here.
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
index 5c55e41c3..ba99c1fc5 100644
--- a/bbot/modules/code_repository.py
+++ b/bbot/modules/code_repository.py
@@ -42,7 +42,7 @@ async def handle_event(self, event):
                         {"url": f"https://{url}"},
                         "CODE_REPOSITORY",
                         tags=platform,
-                        source=event,
+                        parent=event,
                     )
                     repo_event.scope_distance = event.scope_distance
                     await self.emit_event(repo_event)
diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py
index 382644007..6c7bb15cf 100644
--- a/bbot/modules/credshed.py
+++ b/bbot/modules/credshed.py
@@ -75,13 +75,13 @@ async def handle_event(self, event):
             if src:
                 tags = [f"credshed-source-{src}"]
 
-            email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags)
+            email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=tags)
             if email_event is not None:
                 await self.emit_event(email_event)
                 if user:
-                    await self.emit_event(f"{email}:{user}", "USERNAME", source=email_event, tags=tags)
+                    await self.emit_event(f"{email}:{user}", "USERNAME", parent=email_event, tags=tags)
                 if pw:
-                    await self.emit_event(f"{email}:{pw}", "PASSWORD", source=email_event, tags=tags)
+                    await self.emit_event(f"{email}:{pw}", "PASSWORD", parent=email_event, tags=tags)
                 for h_pw in hashes:
                     if h_pw:
-                        await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", source=email_event, tags=tags)
+                        await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", parent=email_event, tags=tags)
diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py
index 94e42d6be..de20f339d 100644
--- a/bbot/modules/deadly/ffuf.py
+++ b/bbot/modules/deadly/ffuf.py
@@ -71,7 +71,7 @@ async def handle_event(self, event):
 
         filters = await self.baseline_ffuf(fixed_url, exts=exts)
         async for r in self.execute_ffuf(self.tempfile, fixed_url, exts=exts, filters=filters):
-            await self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+            await self.emit_event(r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
 
     async def filter_event(self, event):
         if "endpoint" in event.tags:
diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py
index 9234f7494..1898e44d1 100644
--- a/bbot/modules/deadly/nuclei.py
+++ b/bbot/modules/deadly/nuclei.py
@@ -140,19 +140,19 @@ async def handle_batch(self, *events):
         async for severity, template, tags, host, url, name, extracted_results in self.execute_nuclei(nuclei_input):
             # this is necessary because sometimes nuclei is inconsistent about the data returned in the host field
             cleaned_host = temp_target.get(host)
-            source_event = self.correlate_event(events, cleaned_host)
+            parent_event = self.correlate_event(events, cleaned_host)
 
-            if not source_event:
+            if not parent_event:
                 continue
 
             if url == "":
-                url = str(source_event.data)
+                url = str(parent_event.data)
 
             if severity == "INFO" and "tech" in tags:
                 await self.emit_event(
-                    {"technology": str(name).lower(), "url": url, "host": str(source_event.host)},
+                    {"technology": str(name).lower(), "url": url, "host": str(parent_event.host)},
                     "TECHNOLOGY",
-                    source_event,
+                    parent_event,
                 )
                 continue
 
@@ -163,30 +163,30 @@ async def handle_batch(self, *events):
             if severity in ["INFO", "UNKNOWN"]:
                 await self.emit_event(
                     {
-                        "host": str(source_event.host),
+                        "host": str(parent_event.host),
                         "url": url,
                         "description": description_string,
                     },
                     "FINDING",
-                    source_event,
+                    parent_event,
                 )
             else:
                 await self.emit_event(
                     {
                         "severity": severity,
-                        "host": str(source_event.host),
+                        "host": str(parent_event.host),
                         "url": url,
                         "description": description_string,
                     },
                     "VULNERABILITY",
-                    source_event,
+                    parent_event,
                 )
 
     def correlate_event(self, events, host):
         for event in events:
             if host in event:
                 return event
-        self.verbose(f"Failed to correlate nuclei result for {host}. Possible source events:")
+        self.verbose(f"Failed to correlate nuclei result for {host}. Possible parent events:")
         for event in events:
             self.verbose(f" - {event.data}")
 
diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py
index 101f41536..23fce22d7 100644
--- a/bbot/modules/deadly/vhost.py
+++ b/bbot/modules/deadly/vhost.py
@@ -82,9 +82,9 @@ async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=F
             found_vhost_b64 = r["input"]["FUZZ"]
             vhost_dict = {"host": str(event.host), "url": host, "vhost": base64.b64decode(found_vhost_b64).decode()}
             if f"{vhost_dict['vhost']}{basehost}" != event.parsed_url.netloc:
-                await self.emit_event(vhost_dict, "VHOST", source=event)
+                await self.emit_event(vhost_dict, "VHOST", parent=event)
                 if skip_dns_host == False:
-                    await self.emit_event(f"{vhost_dict['vhost']}{basehost}", "DNS_NAME", source=event, tags=["vhost"])
+                    await self.emit_event(f"{vhost_dict['vhost']}{basehost}", "DNS_NAME", parent=event, tags=["vhost"])
 
                 yield vhost_dict["vhost"]
 
diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py
index caa5fb662..0cdba9889 100644
--- a/bbot/modules/dehashed.py
+++ b/bbot/modules/dehashed.py
@@ -48,15 +48,15 @@ async def handle_event(self, event):
                 if db_name:
                     tags = [f"db-{db_name}"]
                 if email:
-                    email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags)
+                    email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=tags)
                     if email_event is not None:
                         await self.emit_event(email_event)
                         if user:
-                            await self.emit_event(f"{email}:{user}", "USERNAME", source=email_event, tags=tags)
+                            await self.emit_event(f"{email}:{user}", "USERNAME", parent=email_event, tags=tags)
                         if pw:
-                            await self.emit_event(f"{email}:{pw}", "PASSWORD", source=email_event, tags=tags)
+                            await self.emit_event(f"{email}:{pw}", "PASSWORD", parent=email_event, tags=tags)
                         if h_pw:
-                            await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", source=email_event, tags=tags)
+                            await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", parent=email_event, tags=tags)
 
     async def query(self, event):
         query = f"domain:{event.data}"
diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py
index 429a55620..4d4240b6f 100644
--- a/bbot/modules/dnsbrute.py
+++ b/bbot/modules/dnsbrute.py
@@ -46,4 +46,4 @@ async def handle_event(self, event):
         query = self.make_query(event)
         self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
         for hostname in await self.helpers.dns.brute(self, query, self.subdomain_list):
-            await self.emit_event(hostname, "DNS_NAME", source=event)
+            await self.emit_event(hostname, "DNS_NAME", parent=event)
diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py
index a02ef7e9e..1a836c971 100644
--- a/bbot/modules/dnsbrute_mutations.py
+++ b/bbot/modules/dnsbrute_mutations.py
@@ -17,7 +17,7 @@ class dnsbrute_mutations(BaseModule):
 
     async def setup(self):
         self.found = {}
-        self.source_events = self.helpers.make_target()
+        self.parent_events = self.helpers.make_target()
         self.max_mutations = self.config.get("max_mutations", 500)
         # 800M bits == 100MB bloom filter == 10M entries before false positives start emerging
         self.mutations_tried = self.helpers.bloom_filter(800000000)
@@ -29,7 +29,7 @@ async def handle_event(self, event):
         self.add_found(event)
 
     def add_found(self, event):
-        self.source_events.add(event)
+        self.parent_events.add(event)
         host = str(event.host)
         if self.helpers.is_subdomain(host):
             subdomain, domain = host.split(".", 1)
@@ -104,19 +104,19 @@ def add_mutation(m):
                         self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(trimmed_found)})")
                         results = await self.helpers.dns.brute(self, query, mutations)
                         for hostname in results:
-                            source_event = self.source_events.get_host(hostname)
-                            if source_event is None:
-                                self.warning(f"Could not correlate source event from: {hostname}")
-                                self.warning(self.source_events._radix.dns_tree.root.children)
-                                self.warning(self.source_events._radix.dns_tree.root.children["com"].children)
+                            parent_event = self.parent_events.get_host(hostname)
+                            if parent_event is None:
+                                self.warning(f"Could not correlate parent event from: {hostname}")
+                                self.warning(self.parent_events._radix.dns_tree.root.children)
+                                self.warning(self.parent_events._radix.dns_tree.root.children["com"].children)
                                 self.warning(
-                                    self.source_events._radix.dns_tree.root.children["com"].children["tesla"].children
+                                    self.parent_events._radix.dns_tree.root.children["com"].children["tesla"].children
                                 )
-                                source_event = self.scan.root_event
+                                parent_event = self.scan.root_event
                             await self.emit_event(
                                 hostname,
                                 "DNS_NAME",
-                                source=source_event,
+                                parent=parent_event,
                                 tags=[f"mutation-{self._mutation_run}"],
                                 abort_if=self.abort_if,
                             )
diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py
index ae4f39c88..5e5d5b315 100644
--- a/bbot/modules/dnscommonsrv.py
+++ b/bbot/modules/dnscommonsrv.py
@@ -160,4 +160,4 @@ async def handle_event(self, event):
         query = self.make_query(event)
         self.verbose(f'Brute-forcing SRV records for "{query}"')
         for hostname in await self.helpers.dns.brute(self, query, common_srvs, type="SRV"):
-            await self.emit_event(hostname, "DNS_NAME", source=event)
+            await self.emit_event(hostname, "DNS_NAME", parent=event)
diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py
index 86c7a07a7..111ac7637 100644
--- a/bbot/modules/docker_pull.py
+++ b/bbot/modules/docker_pull.py
@@ -53,7 +53,7 @@ async def handle_event(self, event):
                 {"path": str(repo_path), "description": f"Docker image repository: {repo_url}"},
                 "FILESYSTEM",
                 tags=["docker", "tarball"],
-                source=event,
+                parent=event,
             )
             codebase_event.scope_distance = event.scope_distance
             await self.emit_event(codebase_event)
diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py
index aedaf7290..d506e877f 100644
--- a/bbot/modules/dockerhub.py
+++ b/bbot/modules/dockerhub.py
@@ -36,7 +36,7 @@ async def handle_org_stub(self, event):
                 site_url = f"{self.site_url}/u/{p}"
                 # emit social event
                 await self.emit_event(
-                    {"platform": "docker", "url": site_url, "profile_name": p}, "SOCIAL", source=event
+                    {"platform": "docker", "url": site_url, "profile_name": p}, "SOCIAL", parent=event
                 )
 
     async def handle_social(self, event):
@@ -44,11 +44,11 @@ async def handle_social(self, event):
         if not username:
             return
         # emit API endpoint to be visited by httpx (for url/email extraction, etc.)
-        await self.emit_event(f"{self.api_url}/users/{username}", "URL_UNVERIFIED", source=event, tags="httpx-safe")
+        await self.emit_event(f"{self.api_url}/users/{username}", "URL_UNVERIFIED", parent=event, tags="httpx-safe")
         self.verbose(f"Searching for docker images belonging to {username}")
         repos = await self.get_repos(username)
         for repo in repos:
-            await self.emit_event({"url": repo}, "CODE_REPOSITORY", tags="docker", source=event)
+            await self.emit_event({"url": repo}, "CODE_REPOSITORY", tags="docker", parent=event)
 
     async def get_repos(self, username):
         repos = []
diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py
index 31cff1468..3b5fbe648 100644
--- a/bbot/modules/emailformat.py
+++ b/bbot/modules/emailformat.py
@@ -19,4 +19,4 @@ async def handle_event(self, event):
             return
         for email in await self.helpers.re.extract_emails(r.text):
             if email.endswith(query):
-                await self.emit_event(email, "EMAIL_ADDRESS", source=event)
+                await self.emit_event(email, "EMAIL_ADDRESS", parent=event)
diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py
index 49cb8a779..d0d696669 100644
--- a/bbot/modules/ffuf_shortnames.py
+++ b/bbot/modules/ffuf_shortnames.py
@@ -112,19 +112,19 @@ def find_delimiter(self, hint):
         return None
 
     async def filter_event(self, event):
-        if event.source.type != "URL":
-            return False, "its source event is not of type URL"
+        if event.parent.type != "URL":
+            return False, "its parent event is not of type URL"
         return True
 
     async def handle_event(self, event):
         filename_hint = re.sub(r"~\d", "", event.parsed_url.path.rsplit(".", 1)[0].split("/")[-1]).lower()
 
-        host = f"{event.source.parsed_url.scheme}://{event.source.parsed_url.netloc}/"
+        host = f"{event.parent.parsed_url.scheme}://{event.parent.parsed_url.netloc}/"
         if host not in self.per_host_collection.keys():
-            self.per_host_collection[host] = [(filename_hint, event.source.data)]
+            self.per_host_collection[host] = [(filename_hint, event.parent.data)]
 
         else:
-            self.per_host_collection[host].append((filename_hint, event.source.data))
+            self.per_host_collection[host].append((filename_hint, event.parent.data))
 
         self.shortname_to_event[filename_hint] = event
 
@@ -148,12 +148,12 @@ async def handle_event(self, event):
             if "shortname-file" in event.tags:
                 for ext in used_extensions:
                     async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"):
-                        await self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+                        await self.emit_event(r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
 
             elif "shortname-directory" in event.tags:
                 async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]):
                     r_url = f"{r['url'].rstrip('/')}/"
-                    await self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+                    await self.emit_event(r_url, "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
 
         if self.config.get("find_delimiters"):
             if "shortname-directory" in event.tags:
@@ -163,7 +163,7 @@ async def handle_event(self, event):
                     self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
                     tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
                     async for r in self.execute_ffuf(tempfile, root_url, prefix=f"{prefix}{delimiter}", exts=["/"]):
-                        await self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"])
+                        await self.emit_event(r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
 
             elif "shortname-file" in event.tags:
                 for ext in used_extensions:
@@ -176,7 +176,7 @@ async def handle_event(self, event):
                             tempfile, root_url, prefix=f"{prefix}{delimiter}", suffix=f".{ext}"
                         ):
                             await self.emit_event(
-                                r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]
+                                r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"]
                             )
 
     async def finish(self):
@@ -208,7 +208,7 @@ async def finish(self):
                                         await self.emit_event(
                                             r["url"],
                                             "URL_UNVERIFIED",
-                                            source=self.shortname_to_event[hint],
+                                            parent=self.shortname_to_event[hint],
                                             tags=[f"status-{r['status']}"],
                                         )
                                 elif "shortname-file" in self.shortname_to_event[hint].tags:
@@ -224,6 +224,6 @@ async def finish(self):
                                             await self.emit_event(
                                                 r["url"],
                                                 "URL_UNVERIFIED",
-                                                source=self.shortname_to_event[hint],
+                                                parent=self.shortname_to_event[hint],
                                                 tags=[f"status-{r['status']}"],
                                             )
diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py
index 41efa75b6..3d9a54f6a 100644
--- a/bbot/modules/fingerprintx.py
+++ b/bbot/modules/fingerprintx.py
@@ -46,10 +46,10 @@ async def handle_batch(self, *events):
             if host and ip:
                 tags.add(f"ip-{ip}")
             if host and port and protocol:
-                source_event = _input.get(port_data)
+                parent_event = _input.get(port_data)
                 protocol_data = {"host": host, "protocol": protocol.upper()}
                 if port:
                     protocol_data["port"] = port
                 if banner:
                     protocol_data["banner"] = banner
-                await self.emit_event(protocol_data, "PROTOCOL", source=source_event, tags=tags)
+                await self.emit_event(protocol_data, "PROTOCOL", parent=parent_event, tags=tags)
diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py
index 2778efdde..a9629beee 100644
--- a/bbot/modules/git_clone.py
+++ b/bbot/modules/git_clone.py
@@ -35,7 +35,7 @@ async def handle_event(self, event):
         if repo_path:
             self.verbose(f"Cloned {repo_url} to {repo_path}")
             codebase_event = self.make_event(
-                {"path": str(repo_path)}, "FILESYSTEM", tags=["git", "folder"], source=event
+                {"path": str(repo_path)}, "FILESYSTEM", tags=["git", "folder"], parent=event
             )
             codebase_event.scope_distance = event.scope_distance
             await self.emit_event(codebase_event)
diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py
index 756645c13..7472d701e 100644
--- a/bbot/modules/github_codesearch.py
+++ b/bbot/modules/github_codesearch.py
@@ -19,12 +19,12 @@ async def setup(self):
     async def handle_event(self, event):
         query = self.make_query(event)
         for repo_url, raw_urls in (await self.query(query)).items():
-            repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", tags="git", source=event)
+            repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", tags="git", parent=event)
             if repo_event is None:
                 continue
             await self.emit_event(repo_event)
             for raw_url in raw_urls:
-                url_event = self.make_event(raw_url, "URL_UNVERIFIED", source=repo_event, tags=["httpx-safe"])
+                url_event = self.make_event(raw_url, "URL_UNVERIFIED", parent=repo_event, tags=["httpx-safe"])
                 if not url_event:
                     continue
                 url_event.scope_distance = repo_event.scope_distance
diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py
index cf836af3e..e5071ea8b 100644
--- a/bbot/modules/github_org.py
+++ b/bbot/modules/github_org.py
@@ -55,7 +55,7 @@ async def handle_event(self, event):
                 self.verbose(f"Searching for repos belonging to user {user}")
                 repos = await self.query_user_repos(user)
             for repo_url in repos:
-                repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", tags="git", source=event)
+                repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", tags="git", parent=event)
                 if not repo_event:
                     continue
                 repo_event.scope_distance = event.scope_distance
@@ -67,7 +67,7 @@ async def handle_event(self, event):
                 org_members = await self.query_org_members(user)
                 for member in org_members:
                     event_data = {"platform": "github", "profile_name": member, "url": f"https://github.com/{member}"}
-                    member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", source=event)
+                    member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", soparenturce=event)
                     if member_event:
                         await self.emit_event(member_event)
 
@@ -83,7 +83,7 @@ async def handle_event(self, event):
                 return
 
             event_data = {"platform": "github", "profile_name": user, "url": f"https://github.com/{user}"}
-            github_org_event = self.make_event(event_data, "SOCIAL", tags="github-org", source=event)
+            github_org_event = self.make_event(event_data, "SOCIAL", tags="github-org", parent=event)
             if github_org_event:
                 github_org_event.scope_distance = event.scope_distance
                 await self.emit_event(github_org_event)
diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 03d8e55f0..a55f99b7a 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -51,7 +51,7 @@ async def handle_event(self, event):
                         },
                         "FILESYSTEM",
                         tags=["textfile"],
-                        source=event,
+                        parent=event,
                     )
                     logfile_event.scope_distance = event.scope_distance
                     await self.emit_event(logfile_event)
diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py
index 3675adfd3..a1bc057d5 100644
--- a/bbot/modules/gitlab.py
+++ b/bbot/modules/gitlab.py
@@ -49,12 +49,12 @@ async def handle_http_response(self, event):
         if "x_gitlab_meta" in headers:
             url = event.parsed_url._replace(path="/").geturl()
             await self.emit_event(
-                {"host": str(event.host), "technology": "GitLab", "url": url}, "TECHNOLOGY", source=event
+                {"host": str(event.host), "technology": "GitLab", "url": url}, "TECHNOLOGY", parent=event
             )
             await self.emit_event(
                 {"host": str(event.host), "description": f"GitLab server at {event.host}"},
                 "FINDING",
-                source=event,
+                parent=event,
             )
 
     async def handle_technology(self, event):
@@ -89,7 +89,7 @@ async def handle_projects_url(self, projects_url, event):
         for project in await self.gitlab_json_request(projects_url):
             project_url = project.get("web_url", "")
             if project_url:
-                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", source=event)
+                code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
                 code_event.scope_distance = event.scope_distance
                 await self.emit_event(code_event)
             namespace = project.get("namespace", {})
@@ -120,7 +120,7 @@ async def handle_namespace(self, namespace, event):
             social_event = self.make_event(
                 {"platform": "gitlab", "profile_name": namespace_path, "url": namespace_url},
                 "SOCIAL",
-                source=event,
+                parent=event,
             )
             social_event.scope_distance = event.scope_distance
             await self.emit_event(social_event)
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index ea8663bb7..a08fc0b47 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -127,8 +127,8 @@ async def handle_batch(self, *events):
             final_url = screenshot["final_url"]
             filename = screenshot["filename"]
             webscreenshot_data = {"filename": filename, "url": final_url}
-            source_event = event_dict[url]
-            await self.emit_event(webscreenshot_data, "WEBSCREENSHOT", source=source_event)
+            parent_event = event_dict[url]
+            await self.emit_event(webscreenshot_data, "WEBSCREENSHOT", parent=parent_event)
 
         # emit URLs
         for url, row in self.new_network_logs.items():
@@ -137,21 +137,21 @@ async def handle_batch(self, *events):
             tags = [f"status-{status_code}", f"ip-{ip}"]
 
             _id = row["url_id"]
-            source_url = self.screenshots_taken[_id]
-            source_event = event_dict[source_url]
-            if self.helpers.is_spider_danger(source_event, url):
+            parent_url = self.screenshots_taken[_id]
+            parent_event = event_dict[parent_url]
+            if self.helpers.is_spider_danger(parent_event, url):
                 tags.append("spider-danger")
             if url and url.startswith("http"):
-                await self.emit_event(url, "URL_UNVERIFIED", source=source_event, tags=tags)
+                await self.emit_event(url, "URL_UNVERIFIED", parent=parent_event, tags=tags)
 
         # emit technologies
         for _, row in self.new_technologies.items():
-            source_id = row["url_id"]
-            source_url = self.screenshots_taken[source_id]
-            source_event = event_dict[source_url]
+            parent_id = row["url_id"]
+            parent_url = self.screenshots_taken[parent_id]
+            parent_event = event_dict[parent_url]
             technology = row["value"]
-            tech_data = {"technology": technology, "url": source_url, "host": str(source_event.host)}
-            await self.emit_event(tech_data, "TECHNOLOGY", source=source_event)
+            tech_data = {"technology": technology, "url": parent_url, "host": str(parent_event.host)}
+            await self.emit_event(tech_data, "TECHNOLOGY", parent=parent_event)
 
     def construct_command(self):
         # base executable
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index c5e1a42f3..98c771057 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -23,7 +23,7 @@ class httpx(BaseModule):
     }
     options_desc = {
         "threads": "Number of httpx threads to use",
-        "in_scope_only": "Only visit web resources that are in scope.",
+        "in_scope_only": "Only visit web reparents that are in scope.",
         "version": "httpx version",
         "max_response_size": "Max response size in bytes",
         "store_responses": "Save raw HTTP responses to scan folder",
@@ -145,15 +145,15 @@ async def handle_batch(self, *events):
                 self.debug(f'No HTTP status code for "{url}"')
                 continue
 
-            source_event = stdin.get(j.get("input", ""), None)
+            parent_event = stdin.get(j.get("input", ""), None)
 
-            if source_event is None:
-                self.warning(f"Unable to correlate source event from: {line}")
+            if parent_event is None:
+                self.warning(f"Unable to correlate parent event from: {line}")
                 continue
 
             # discard 404s from unverified URLs
             path = j.get("path", "/")
-            if source_event.type == "URL_UNVERIFIED" and status_code in (404,) and path != "/":
+            if parent_event.type == "URL_UNVERIFIED" and status_code in (404,) and path != "/":
                 self.debug(f'Discarding 404 from "{url}"')
                 continue
 
@@ -169,9 +169,9 @@ async def handle_batch(self, *events):
             title = self.helpers.tagify(j.get("title", ""), maxlen=30)
             if title:
                 tags.append(f"http-title-{title}")
-            url_event = self.make_event(url, "URL", source_event, tags=tags)
+            url_event = self.make_event(url, "URL", parent_event, tags=tags)
             if url_event:
-                if url_event != source_event:
+                if url_event != parent_event:
                     await self.emit_event(url_event)
                 # HTTP response
                 await self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags)
diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py
index 7939487fd..84933f091 100644
--- a/bbot/modules/internal/cloud.py
+++ b/bbot/modules/internal/cloud.py
@@ -34,7 +34,7 @@ async def handle_event(self, event, kwargs):
         for provider in self.helpers.cloud.providers.values():
             provider_name = provider.name.lower()
             base_kwargs = dict(
-                source=event, tags=[f"{provider.provider_type}-{provider_name}"], _provider=provider_name
+                parent=event, tags=[f"{provider.provider_type}-{provider_name}"], _provider=provider_name
             )
             # loop through the provider's regex signatures, if any
             for event_type, sigs in provider.signatures.items():
diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index f501fb08f..12861eda7 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -168,19 +168,19 @@ async def handle_event(self, event, kwargs):
             event.type = "DNS_NAME_UNRESOLVED"
 
         # speculate DNS_NAMES and IP_ADDRESSes from other event types
-        source_event = event
+        parent_event = event
         if (
             event.host
             and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE")
             and not ((event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate"))
         ):
-            source_event = self.scan.make_event(event.host, "DNS_NAME", module=self.host_module, source=event)
+            parent_event = self.scan.make_event(event.host, "DNS_NAME", module=self.host_module, parent=event)
             # only emit the event if it's not already in the parent chain
-            if source_event is not None and (source_event.always_emit or source_event not in event.get_sources()):
-                source_event.scope_distance = event.scope_distance
+            if parent_event is not None and (parent_event.always_emit or parent_event not in event.get_parents()):
+                parent_event.scope_distance = event.scope_distance
                 if "target" in event.tags:
-                    source_event.add_tag("target")
-                await self.emit_event(source_event)
+                    parent_event.add_tag("target")
+                await self.emit_event(parent_event)
 
         # emit DNS children
         if emit_children:
@@ -192,7 +192,7 @@ async def handle_event(self, event, kwargs):
                     module._priority = 4
                     for record in records:
                         try:
-                            child_event = self.scan.make_event(record, "DNS_NAME", module=module, source=source_event)
+                            child_event = self.scan.make_event(record, "DNS_NAME", module=module, parent=parent_event)
                             # if it's a hostname and it's only one hop away, mark it as affiliate
                             if child_event.type == "DNS_NAME" and child_event.scope_distance == 1:
                                 child_event.add_tag("affiliate")
@@ -200,7 +200,7 @@ async def handle_event(self, event, kwargs):
                                 dns_child_events.append(child_event)
                         except ValidationError as e:
                             self.warning(
-                                f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}'
+                                f'Event validation failed for DNS child of {parent_event}: "{record}" ({rdtype}): {e}'
                             )
             for child_event in dns_child_events:
                 self.debug(f"Queueing DNS child for {event}: {child_event}")
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 066d095dd..60d9ad73b 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -52,7 +52,7 @@ async def search(self, content, event, **kwargs):
                 await self.report(domain, event, **kwargs)
 
     async def report(self, domain, event, **kwargs):
-        await self.excavate.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"])
+        await self.excavate.emit_event(domain, "DNS_NAME", parent=event, tags=["affiliate"])
 
 
 class HostnameExtractor(BaseExtractor):
@@ -64,7 +64,7 @@ def __init__(self, excavate):
         super().__init__(excavate)
 
     async def report(self, result, name, event, **kwargs):
-        await self.excavate.emit_event(result, "DNS_NAME", source=event)
+        await self.excavate.emit_event(result, "DNS_NAME", parent=event)
 
 
 class URLExtractor(BaseExtractor):
@@ -173,7 +173,7 @@ async def report(self, result, name, event, **kwargs):
             await self.excavate.emit_event(
                 event_data,
                 "FINDING",
-                source=event,
+                parent=event,
                 abort_if=abort_if,
             )
             protocol_data = {"protocol": parsed_uri.scheme, "host": str(host)}
@@ -182,12 +182,12 @@ async def report(self, result, name, event, **kwargs):
             await self.excavate.emit_event(
                 protocol_data,
                 "PROTOCOL",
-                source=event,
+                parent=event,
                 abort_if=abort_if,
             )
             return
 
-        return self.excavate.make_event(result, "URL_UNVERIFIED", source=event)
+        return self.excavate.make_event(result, "URL_UNVERIFIED", parent=event)
 
 
 class EmailExtractor(BaseExtractor):
@@ -199,7 +199,7 @@ async def report(self, result, name, event, **kwargs):
         tld = result.split(".")[-1]
         if tld not in self.tld_blacklist:
             self.excavate.debug(f"Found email address [{result}] from parsing [{event.data.get('url')}]")
-            await self.excavate.emit_event(result, "EMAIL_ADDRESS", source=event)
+            await self.excavate.emit_event(result, "EMAIL_ADDRESS", parent=event)
 
 
 class ErrorExtractor(BaseExtractor):
@@ -226,7 +226,7 @@ async def report(self, result, name, event, **kwargs):
         await self.excavate.emit_event(
             {"host": str(event.host), "url": event.data.get("url", ""), "description": description},
             "FINDING",
-            source=event,
+            parent=event,
         )
 
 
@@ -381,8 +381,8 @@ async def handle_event(self, event):
                         url_event = self.make_event(location, "URL_UNVERIFIED", event, tags="affiliate")
                         if url_event is not None:
                             # inherit web spider distance from parent (don't increment)
-                            source_web_spider_distance = getattr(event, "web_spider_distance", 0)
-                            url_event.web_spider_distance = source_web_spider_distance
+                            parent_web_spider_distance = getattr(event, "web_spider_distance", 0)
+                            url_event.web_spider_distance = parent_web_spider_distance
                             await self.emit_event(url_event)
                     else:
                         self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}")
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 9f39598d2..601b9ec81 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -75,13 +75,13 @@ async def handle_event(self, event):
             ips = list(net)
             random.shuffle(ips)
             for ip in ips:
-                await self.emit_event(ip, "IP_ADDRESS", source=event, internal=True)
+                await self.emit_event(ip, "IP_ADDRESS", parent=event, internal=True)
 
         # parent domains
         if event.type == "DNS_NAME":
             parent = self.helpers.parent_domain(event.data)
             if parent != event.data:
-                await self.emit_event(parent, "DNS_NAME", source=event, internal=True)
+                await self.emit_event(parent, "DNS_NAME", parent=event, internal=True)
 
         # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
         event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
@@ -94,7 +94,7 @@ async def handle_event(self, event):
                 await self.emit_event(
                     self.helpers.make_netloc(event.host, event.port),
                     "OPEN_TCP_PORT",
-                    source=event,
+                    parent=event,
                     internal=True,
                     quick=(event.type == "URL"),
                 )
@@ -103,11 +103,11 @@ async def handle_event(self, event):
         if event.type == "URL":
             url_parents = self.helpers.url_parents(event.data)
             for up in url_parents:
-                url_event = self.make_event(f"{up}/", "URL_UNVERIFIED", source=event)
+                url_event = self.make_event(f"{up}/", "URL_UNVERIFIED", parent=event)
                 if url_event is not None:
                     # inherit web spider distance from parent (don't increment)
-                    source_web_spider_distance = getattr(event, "web_spider_distance", 0)
-                    url_event.web_spider_distance = source_web_spider_distance
+                    parent_web_spider_distance = getattr(event, "web_spider_distance", 0)
+                    url_event.web_spider_distance = parent_web_spider_distance
                     await self.emit_event(url_event)
 
         # speculate URL_UNVERIFIED from URL or any event with "url" attribute
@@ -118,11 +118,11 @@ async def handle_event(self, event):
                 url = event.data
             else:
                 url = event.data["url"]
-            if not any(e.type == "URL_UNVERIFIED" and e.data == url for e in event.get_sources()):
+            if not any(e.type == "URL_UNVERIFIED" and e.data == url for e in event.get_parents()):
                 tags = None
-                if self.helpers.is_spider_danger(event.source, url):
+                if self.helpers.is_spider_danger(event.parent, url):
                     tags = ["spider-danger"]
-                await self.emit_event(url, "URL_UNVERIFIED", tags=tags, source=event)
+                await self.emit_event(url, "URL_UNVERIFIED", tags=tags, parent=event)
 
         # from hosts
         if speculate_open_ports:
@@ -137,7 +137,7 @@ async def handle_event(self, event):
                     await self.emit_event(
                         self.helpers.make_netloc(event.data, port),
                         "OPEN_TCP_PORT",
-                        source=event,
+                        parent=event,
                         internal=True,
                         quick=True,
                     )
@@ -164,7 +164,7 @@ async def handle_event(self, event):
             stub_hash = hash(stub)
             if stub_hash not in self.org_stubs_seen:
                 self.org_stubs_seen.add(stub_hash)
-                stub_event = self.make_event(stub, "ORG_STUB", source=event)
+                stub_event = self.make_event(stub, "ORG_STUB", parent=event)
                 if stub_event:
                     if event.scope_distance > 0:
                         stub_event.scope_distance = event.scope_distance
@@ -174,7 +174,7 @@ async def handle_event(self, event):
         if event.type == "USERNAME":
             email = event.data.split(":", 1)[-1]
             if validators.soft_validate(email, "email"):
-                email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=["affiliate"])
+                email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=["affiliate"])
                 if email_event:
                     email_event.scope_distance = event.scope_distance
                     await self.emit_event(email_event)
diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py
index 847db0c7a..9a7e30493 100644
--- a/bbot/modules/internetdb.py
+++ b/bbot/modules/internetdb.py
@@ -90,12 +90,12 @@ async def _parse_response(self, data: dict, event):
         data: dict  # has keys: cpes, hostnames, ip, ports, tags, vulns
         # ip is a string, ports is a list of ports, the rest is a list of strings
         for hostname in data.get("hostnames", []):
-            await self.emit_event(hostname, "DNS_NAME", source=event)
+            await self.emit_event(hostname, "DNS_NAME", parent=event)
         for cpe in data.get("cpes", []):
-            await self.emit_event({"technology": cpe, "host": str(event.host)}, "TECHNOLOGY", source=event)
+            await self.emit_event({"technology": cpe, "host": str(event.host)}, "TECHNOLOGY", parent=event)
         for port in data.get("ports", []):
             await self.emit_event(
-                self.helpers.make_netloc(event.data, port), "OPEN_TCP_PORT", source=event, internal=True, quick=True
+                self.helpers.make_netloc(event.data, port), "OPEN_TCP_PORT", parent=event, internal=True, quick=True
             )
         vulns = data.get("vulns", [])
         if vulns:
@@ -103,7 +103,7 @@ async def _parse_response(self, data: dict, event):
             await self.emit_event(
                 {"description": f"Shodan reported verified vulnerabilities: {vulns_str}", "host": str(event.host)},
                 "FINDING",
-                source=event,
+                parent=event,
             )
 
     def get_ip(self, event):
diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py
index ddeb76163..ce9c64bc0 100644
--- a/bbot/modules/masscan.py
+++ b/bbot/modules/masscan.py
@@ -36,7 +36,7 @@ class masscan(portscanner):
             "ignore_errors": True,
         },
         {
-            "name": "Download masscan source code",
+            "name": "Download masscan parent code",
             "git": {
                 "repo": "https://github.com/robertdavidgraham/masscan.git",
                 "dest": "#{BBOT_TEMP}/masscan",
@@ -186,28 +186,28 @@ async def process_output(self, line, result_callback):
             if proto == "" or port_number == "":
                 continue
             result = str(ip)
-            source = None
+            parent = None
             with suppress(KeyError):
-                source = self.alive_hosts[ip]
+                parent = self.alive_hosts[ip]
             if proto != "icmp":
                 result = self.helpers.make_netloc(result, port_number)
-                if source is None:
-                    source = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip))
-                    if not source:
+                if parent is None:
+                    parent = self.make_event(ip, "IP_ADDRESS", parent=self.get_parent_event(ip))
+                    if not parent:
                         continue
-                    await self.emit_event(source)
-            await result_callback(result, source=source)
+                    await self.emit_event(parent)
+            await result_callback(result, parent=parent)
 
-    async def append_alive_host(self, host, source):
-        host_event = self.make_event(host, "IP_ADDRESS", source=self.get_source_event(host))
+    async def append_alive_host(self, host, parent):
+        host_event = self.make_event(host, "IP_ADDRESS", parent=self.get_parent_event(host))
         if host_event:
             self.alive_hosts[host] = host_event
             self._write_ping_result(host)
             await self.emit_event(host_event)
 
-    async def emit_open_tcp_port(self, data, source):
+    async def emit_open_tcp_port(self, data, parent):
         self._write_syn_result(data)
-        await self.emit_event(data, "OPEN_TCP_PORT", source=source)
+        await self.emit_event(data, "OPEN_TCP_PORT", parent=parent)
 
     async def emit_from_cache(self):
         ip_events = {}
@@ -221,7 +221,7 @@ async def emit_from_cache(self):
             for ip in cached_pings:
                 if self.scan.stopping:
                     break
-                ip_event = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip))
+                ip_event = self.make_event(ip, "IP_ADDRESS", parent=self.get_parent_event(ip))
                 if ip_event:
                     ip_events[ip] = ip_event
                     await self.emit_event(ip_event)
@@ -237,22 +237,22 @@ async def emit_from_cache(self):
                     break
                 host, port = self.helpers.split_host_port(line)
                 host = str(host)
-                source_event = ip_events.get(host)
-                if source_event is None:
-                    self.verbose(f"Source event not found for {line}")
-                    source_event = self.make_event(line, "IP_ADDRESS", source=self.get_source_event(line))
-                    if not source_event:
+                parent_event = ip_events.get(host)
+                if parent_event is None:
+                    self.verbose(f"Parent event not found for {line}")
+                    parent_event = self.make_event(line, "IP_ADDRESS", parent=self.get_parent_event(line))
+                    if not parent_event:
                         continue
-                    await self.emit_event(source_event)
-                await self.emit_event(line, "OPEN_TCP_PORT", source=source_event)
+                    await self.emit_event(parent_event)
+                await self.emit_event(line, "OPEN_TCP_PORT", parent=parent_event)
 
-    def get_source_event(self, host):
-        source_event = self.scan.target.get(host)
-        if source_event is None:
-            source_event = self.scan.whitelist.get(host)
-            if source_event is None:
-                source_event = self.scan.root_event
-        return source_event
+    def get_parent_event(self, host):
+        parent_event = self.scan.target.get(host)
+        if parent_event is None:
+            parent_event = self.scan.whitelist.get(host)
+            if parent_event is None:
+                parent_event = self.scan.root_event
+        return parent_event
 
     async def cleanup(self):
         if self.ping_first:
diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py
index 4374155ab..035708dec 100644
--- a/bbot/modules/nmap.py
+++ b/bbot/modules/nmap.py
@@ -41,21 +41,21 @@ async def handle_batch(self, *events):
         try:
             await self.run_process(command, sudo=True)
             for host in self.parse_nmap_xml(output_file):
-                source_event = None
+                parent_event = None
                 for h in [host.address] + host.hostnames:
-                    source_event = target.get_host(h)
-                    if source_event is not None:
+                    parent_event = target.get_host(h)
+                    if parent_event is not None:
                         break
-                if source_event is None:
-                    self.warning(f"Failed to correlate source event from {host}")
-                    source_event = self.scan.root_event
+                if parent_event is None:
+                    self.warning(f"Failed to correlate parent event from {host}")
+                    parent_event = self.scan.root_event
                 for port in host.open_ports:
                     port_number = int(port.split("/")[0])
                     netloc = self.helpers.make_netloc(host.address, port_number)
-                    await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event)
+                    await self.emit_event(netloc, "OPEN_TCP_PORT", parent=parent_event)
                     for hostname in host.hostnames:
                         netloc = self.helpers.make_netloc(hostname, port_number)
-                        await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event)
+                        await self.emit_event(netloc, "OPEN_TCP_PORT", parent=parent_event)
         finally:
             output_file.unlink(missing_ok=True)
 
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index b08fe4784..3f767d4e2 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -94,11 +94,11 @@ async def handle_event(self, event):
                             "description": f"NTLM AUTH: {result}",
                         },
                         "FINDING",
-                        source=event,
+                        parent=event,
                     )
                     fqdn = result.get("FQDN", "")
                     if fqdn:
-                        await self.emit_event(fqdn, "DNS_NAME", source=event)
+                        await self.emit_event(fqdn, "DNS_NAME", parent=event)
                     break
 
     async def filter_event(self, event):
diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py
index fd6188acd..502bdbf2a 100644
--- a/bbot/modules/oauth.py
+++ b/bbot/modules/oauth.py
@@ -63,13 +63,13 @@ async def handle_event(self, event):
                         "url": url,
                     },
                     "FINDING",
-                    source=event,
+                    parent=event,
                 )
                 if finding_event:
                     finding_event.source_domain = source_domain
                     await self.emit_event(finding_event)
                 url_event = self.make_event(
-                    token_endpoint, "URL_UNVERIFIED", source=event, tags=["affiliate", "oauth-token-endpoint"]
+                    token_endpoint, "URL_UNVERIFIED", parent=event, tags=["affiliate", "oauth-token-endpoint"]
                 )
                 if url_event:
                     url_event.source_domain = source_domain
@@ -77,7 +77,7 @@ async def handle_event(self, event):
             for result in oidc_results:
                 if result not in (domain, event.data):
                     event_type = "URL_UNVERIFIED" if self.helpers.is_url(result) else "DNS_NAME"
-                    await self.emit_event(result, event_type, source=event, tags=["affiliate"])
+                    await self.emit_event(result, event_type, parent=event, tags=["affiliate"])
 
         for oauth_task in oauth_tasks:
             url = await oauth_task
@@ -89,7 +89,7 @@ async def handle_event(self, event):
                         "url": url,
                     },
                     "FINDING",
-                    source=event,
+                    parent=event,
                 )
                 if oauth_finding:
                     oauth_finding.source_domain = source_domain
diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py
index 0372dc1bd..78fc5cbec 100644
--- a/bbot/modules/output/asset_inventory.py
+++ b/bbot/modules/output/asset_inventory.py
@@ -179,23 +179,23 @@ async def finish(self):
                         self.add_custom_headers(list(asset.custom_fields))
                         if not is_ip(asset.host):
                             host_event = self.make_event(
-                                asset.host, "DNS_NAME", source=self.scan.root_event, raise_error=True
+                                asset.host, "DNS_NAME", parent=self.scan.root_event, raise_error=True
                             )
                             await self.emit_event(host_event)
                             for port in asset.ports:
                                 netloc = self.helpers.make_netloc(asset.host, port)
-                                open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", source=host_event)
+                                open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", parent=host_event)
                                 if open_port_event:
                                     await self.emit_event(open_port_event)
                         else:
                             for ip in asset.ip_addresses:
                                 ip_event = self.make_event(
-                                    ip, "IP_ADDRESS", source=self.scan.root_event, raise_error=True
+                                    ip, "IP_ADDRESS", parent=self.scan.root_event, raise_error=True
                                 )
                                 await self.emit_event(ip_event)
                                 for port in asset.ports:
                                     netloc = self.helpers.make_netloc(ip, port)
-                                    open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", source=ip_event)
+                                    open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", parent=ip_event)
                                     if open_port_event:
                                         await self.emit_event(open_port_event)
             else:
diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py
index 2b0548ea9..f72356a4e 100644
--- a/bbot/modules/output/neo4j.py
+++ b/bbot/modules/output/neo4j.py
@@ -53,7 +53,7 @@ async def setup(self):
 
     async def handle_event(self, event):
         # create events
-        src_id = await self.merge_event(event.get_source(), id_only=True)
+        src_id = await self.merge_event(event.get_parent(), id_only=True)
         dst_id = await self.merge_event(event)
         # create relationship
         cypher = f"""
diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py
index 246704cc5..24f914a3e 100644
--- a/bbot/modules/output/web_report.py
+++ b/bbot/modules/output/web_report.py
@@ -38,32 +38,32 @@ async def handle_event(self, event):
             host = f"{parsed.scheme}://{parsed.netloc}/"
             if host not in self.web_assets.keys():
                 self.web_assets[host] = {"URL": []}
-            source_chain = []
+            parent_chain = []
 
-            current_parent = event.source
+            current_parent = event.parent
             while not current_parent.type == "SCAN":
-                source_chain.append(
+                parent_chain.append(
                     f" ({current_parent.module})---> [{current_parent.type}]:{html.escape(current_parent.pretty_string)}"
                 )
-                current_parent = current_parent.source
+                current_parent = current_parent.parent
 
-            source_chain.reverse()
-            source_chain_text = (
-                "".join(source_chain)
+            parent_chain.reverse()
+            parent_chain_text = (
+                "".join(parent_chain)
                 + f" ({event.module})---> "
                 + f"[{event.type}]:{html.escape(event.pretty_string)}"
             )
-            self.web_assets[host]["URL"].append(f"**{html.escape(event.data)}**: {source_chain_text}")
+            self.web_assets[host]["URL"].append(f"**{html.escape(event.data)}**: {parent_chain_text}")
 
         else:
-            current_parent = event.source
+            current_parent = event.parent
             parsed = None
             while 1:
                 if current_parent.type == "URL":
                     parsed = current_parent.parsed_url
                     break
-                current_parent = current_parent.source
-                if current_parent.source.type == "SCAN":
+                current_parent = current_parent.parent
+                if current_parent.parent.type == "SCAN":
                     break
             if parsed:
                 host = f"{parsed.scheme}://{parsed.netloc}/"
diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py
index 348d0dc28..e1a659543 100644
--- a/bbot/modules/postman.py
+++ b/bbot/modules/postman.py
@@ -26,7 +26,7 @@ async def handle_event(self, event):
         query = self.make_query(event)
         self.verbose(f"Searching for any postman workspaces, collections, requests belonging to {query}")
         for url in await self.query(query):
-            await self.emit_event(url, "URL_UNVERIFIED", source=event, tags="httpx-safe")
+            await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags="httpx-safe")
 
     async def query(self, query):
         interesting_urls = []
diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py
index 982c76584..66d92dea1 100644
--- a/bbot/modules/report/asn.py
+++ b/bbot/modules/report/asn.py
@@ -41,12 +41,12 @@ async def handle_event(self, event):
                 for asn in asns:
                     emails = asn.pop("emails", [])
                     self.cache_put(asn)
-                    asn_event = self.make_event(asn, "ASN", source=event)
+                    asn_event = self.make_event(asn, "ASN", parent=event)
                     if not asn_event:
                         continue
                     await self.emit_event(asn_event)
                     for email in emails:
-                        await self.emit_event(email, "EMAIL_ADDRESS", source=asn_event)
+                        await self.emit_event(email, "EMAIL_ADDRESS", parent=asn_event)
 
     async def report(self):
         asn_data = sorted(self.asn_cache.items(), key=lambda x: self.asn_counts[x[0]], reverse=True)
diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py
index f23b89bf7..57649e954 100644
--- a/bbot/modules/robots.py
+++ b/bbot/modules/robots.py
@@ -48,4 +48,4 @@ async def handle_event(self, event):
                         tags = []
                         if self.helpers.is_spider_danger(event, unverified_url):
                             tags.append("spider-danger")
-                        await self.emit_event(unverified_url, "URL_UNVERIFIED", source=event, tags=tags)
+                        await self.emit_event(unverified_url, "URL_UNVERIFIED", parent=event, tags=tags)
diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py
index 38a2b644f..7c9c4aaa8 100644
--- a/bbot/modules/secretsdb.py
+++ b/bbot/modules/secretsdb.py
@@ -57,7 +57,7 @@ async def handle_event(self, event):
             await self.emit_event(
                 event_data,
                 "FINDING",
-                source=event,
+                parent=event,
             )
 
     def search_data(self, resp_body, resp_headers):
diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py
index 35c010ea1..a06cbdbfa 100644
--- a/bbot/modules/skymem.py
+++ b/bbot/modules/skymem.py
@@ -23,7 +23,7 @@ async def handle_event(self, event):
         if not r:
             return
         for email in await self.helpers.re.extract_emails(r.text):
-            await self.emit_event(email, "EMAIL_ADDRESS", source=event)
+            await self.emit_event(email, "EMAIL_ADDRESS", parent=event)
 
         # iterate through other pages
         domain_ids = await self.helpers.re.findall(self.next_page_regex, r.text)
@@ -35,7 +35,7 @@ async def handle_event(self, event):
             if not r2:
                 continue
             for email in await self.helpers.re.extract_emails(r2.text):
-                await self.emit_event(email, "EMAIL_ADDRESS", source=event)
+                await self.emit_event(email, "EMAIL_ADDRESS", parent=event)
             pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text)
             if not pages:
                 break
diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py
index 8ee344452..9889e92cb 100644
--- a/bbot/modules/smuggler.py
+++ b/bbot/modules/smuggler.py
@@ -42,5 +42,5 @@ async def handle_event(self, event):
                     await self.emit_event(
                         {"host": str(event.host), "url": event.data, "description": description},
                         "FINDING",
-                        source=event,
+                        parent=event,
                     )
diff --git a/bbot/modules/social.py b/bbot/modules/social.py
index 3688d7402..c1c4e121a 100644
--- a/bbot/modules/social.py
+++ b/bbot/modules/social.py
@@ -40,7 +40,7 @@ async def handle_event(self, event):
                 social_event = self.make_event(
                     {"platform": platform, "url": f"https://{url}", "profile_name": profile_name},
                     "SOCIAL",
-                    source=event,
+                    parent=event,
                 )
                 social_event.scope_distance = event.scope_distance
                 await self.emit_event(social_event)
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 42f34d23e..19e07ee1a 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -29,7 +29,7 @@ async def setup(self):
 
         # sometimes we run into a server with A LOT of SANs
         # these are usually stupid and useless, so we abort based on a different threshold
-        # depending on whether the source event is in scope
+        # depending on whether the parent event is in scope
         self.in_scope_abort_threshold = 50
         self.out_of_scope_abort_threshold = 10
 
@@ -77,9 +77,9 @@ async def handle_event(self, event):
                     if event_data is not None and event_data != event:
                         self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]")
                         try:
-                            ssl_event = self.make_event(event_data, event_type, source=event, raise_error=True)
-                            source_event = ssl_event.get_source()
-                            if source_event.scope_distance == 0:
+                            ssl_event = self.make_event(event_data, event_type, parent=event, raise_error=True)
+                            parent_event = ssl_event.get_parent()
+                            if parent_event.scope_distance == 0:
                                 tags = ["affiliate"]
                             else:
                                 tags = None
@@ -90,8 +90,8 @@ async def handle_event(self, event):
                             self.debug(f"Invalid data at {host}:{port}: {e}")
 
     def on_success_callback(self, event):
-        source_scope_distance = event.get_source().scope_distance
-        if source_scope_distance == 0 and event.scope_distance > 0:
+        parent_scope_distance = event.get_parent().scope_distance
+        if parent_scope_distance == 0 and event.scope_distance > 0:
             event.add_tag("affiliate")
 
     async def visit_host(self, host, port):
diff --git a/bbot/modules/templates/bucket.py b/bbot/modules/templates/bucket.py
index c1fece0a6..7b6a7ee61 100644
--- a/bbot/modules/templates/bucket.py
+++ b/bbot/modules/templates/bucket.py
@@ -53,7 +53,7 @@ async def handle_dns_name(self, event):
                 bucket_name = d.join(split)
                 buckets.add(bucket_name)
         async for bucket_name, url, tags in self.brute_buckets(buckets, permutations=self.permutations):
-            await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags)
+            await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", parent=event, tags=tags)
 
     async def handle_storage_bucket(self, event):
         url = event.data["url"]
@@ -62,12 +62,12 @@ async def handle_storage_bucket(self, event):
             description, tags = await self._check_bucket_open(bucket_name, url)
             if description:
                 event_data = {"host": event.host, "url": url, "description": description}
-                await self.emit_event(event_data, "FINDING", source=event, tags=tags)
+                await self.emit_event(event_data, "FINDING", parent=event, tags=tags)
 
         async for bucket_name, url, tags in self.brute_buckets(
             [bucket_name], permutations=self.permutations, omit_base=True
         ):
-            await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags)
+            await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", parent=event, tags=tags)
 
     async def brute_buckets(self, buckets, permutations=False, omit_base=False):
         buckets = set(buckets)
diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index a00de5a15..b28c73bd2 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -53,7 +53,7 @@ async def handle_event(self, event):
                 data = {
                     "severity": "High",
                     "description": f"Verified Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
-                    "host": str(event.source.host),
+                    "host": str(event.parent.host),
                 }
                 if description:
                     data["description"] += f" Description: [{description}]"
@@ -61,7 +61,7 @@ async def handle_event(self, event):
             else:
                 data = {
                     "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
-                    "host": str(event.source.host),
+                    "host": str(event.parent.host),
                 }
                 if description:
                     data["description"] += f" Description: [{description}]"
diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py
index ca6ea1bad..784fe6c4b 100644
--- a/bbot/modules/url_manipulation.py
+++ b/bbot/modules/url_manipulation.py
@@ -77,7 +77,7 @@ async def handle_event(self, event):
                                 await self.emit_event(
                                     {"description": description, "host": str(event.host), "url": event.data},
                                     "FINDING",
-                                    source=event,
+                                    parent=event,
                                 )
                         else:
                             self.debug(f"Status code changed to {str(subject_response.status_code)}, ignoring")
diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py
index 4c3811af0..eb27e807e 100644
--- a/bbot/modules/urlscan.py
+++ b/bbot/modules/urlscan.py
@@ -20,22 +20,22 @@ async def setup(self):
     async def handle_event(self, event):
         query = self.make_query(event)
         for domain, url in await self.query(query):
-            source_event = event
+            parent_event = event
             if domain and domain != query:
-                domain_event = self.make_event(domain, "DNS_NAME", source=event)
+                domain_event = self.make_event(domain, "DNS_NAME", parent=event)
                 if domain_event:
                     if str(domain_event.host).endswith(query) and not str(domain_event.host) == str(event.host):
                         await self.emit_event(domain_event, abort_if=self.abort_if)
-                        source_event = domain_event
+                        parent_event = domain_event
             if url:
-                url_event = self.make_event(url, "URL_UNVERIFIED", source=source_event)
+                url_event = self.make_event(url, "URL_UNVERIFIED", parent=parent_event)
                 if url_event:
                     if str(url_event.host).endswith(query):
                         if self.urls:
                             await self.emit_event(url_event, abort_if=self.abort_if)
                         else:
                             await self.emit_event(
-                                str(url_event.host), "DNS_NAME", source=event, abort_if=self.abort_if
+                                str(url_event.host), "DNS_NAME", parent=event, abort_if=self.abort_if
                             )
                     else:
                         self.debug(f"{url_event.host} does not match {query}")
diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py
index ddf941cbc..e21838eaa 100644
--- a/bbot/modules/wafw00f.py
+++ b/bbot/modules/wafw00f.py
@@ -38,7 +38,7 @@ async def handle_event(self, event):
         waf_detections = await self.helpers.run_in_executor(WW.identwaf)
         if waf_detections:
             for waf in waf_detections:
-                await self.emit_event({"host": str(event.host), "url": url, "waf": waf}, "WAF", source=event)
+                await self.emit_event({"host": str(event.host), "url": url, "waf": waf}, "WAF", parent=event)
         else:
             if self.config.get("generic_detect") == True:
                 generic = await self.helpers.run_in_executor(WW.genericdetect)
@@ -51,5 +51,5 @@ async def handle_event(self, event):
                             "info": WW.knowledge["generic"]["reason"],
                         },
                         "WAF",
-                        source=event,
+                        parent=event,
                     )
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index c3210d65d..eddd5f9a4 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -50,8 +50,8 @@ async def init_events(self, events=None):
                 event._dummy = False
                 event.web_spider_distance = 0
                 event.scan = self.scan
-                if event.source is None:
-                    event.source = self.scan.root_event
+                if event.parent is None:
+                    event.parent = self.scan.root_event
                 if event.module is None:
                     event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
                 self.verbose(f"Target: {event}")
@@ -64,16 +64,16 @@ async def handle_event(self, event, kwargs):
         if event._dummy:
             return False, "cannot emit dummy event"
 
-        # don't accept events with self as source
-        if (not event.type == "SCAN") and (event == event.get_source()):
-            return False, "event's source is itself"
+        # don't accept events with self as parent
+        if (not event.type == "SCAN") and (event == event.get_parent()):
+            return False, "event's parent is itself"
 
         # don't accept duplicates
         if (not event._graph_important) and self.is_incoming_duplicate(event, add=True):
             return False, "event was already emitted by its module"
 
         # update event's scope distance based on its parent
-        event.scope_distance = event.source.scope_distance + 1
+        event.scope_distance = event.parent.scope_distance + 1
 
         # special handling of URL extensions
         url_extension = getattr(event, "url_extension", None)
@@ -195,15 +195,15 @@ async def handle_event(self, event, kwargs):
 
         # if we discovered something interesting from an internal event,
         # make sure we preserve its chain of parents
-        source = event.source
-        if source.internal and ((not event.internal) or event._graph_important):
-            source_in_report_distance = source.scope_distance <= self.scan.scope_report_distance
-            if source_in_report_distance:
-                source.internal = False
-            if not source._graph_important:
-                source._graph_important = True
-                log.debug(f"Re-queuing internal event {source} with parent {event}")
-                await self.emit_event(source)
+        parent = event.parent
+        if parent.internal and ((not event.internal) or event._graph_important):
+            parent_in_report_distance = parent.scope_distance <= self.scan.scope_report_distance
+            if parent_in_report_distance:
+                parent.internal = False
+            if not parent._graph_important:
+                parent._graph_important = True
+                log.debug(f"Re-queuing internal event {parent} with parent {event}")
+                await self.emit_event(parent)
 
         abort_result = False
         if callable(abort_if):
@@ -244,7 +244,7 @@ async def forward_event(self, event, kwargs):
     def is_outgoing_duplicate(self, event, add=False):
         """
         Calculate whether an event is a duplicate in the context of the whole scan,
-        This will return True if the same event (irregardless of its source module) has been emitted before.
+        This will return True if the same event (regardless of its parent module) has been emitted before.
 
         TODO: Allow modules to use this for custom deduplication such as on a per-host or per-domain basis.
         """
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index ee17ef6a7..f48a4f51c 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -908,7 +908,7 @@ def root_event(self):
               "scope_distance": 0,
               "scan": "SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54",
               "timestamp": 1694548779.616255,
-              "source": "SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54",
+              "parent": "SCAN:1188928d942ace8e3befae0bdb9c3caa22705f54",
               "tags": [
                 "distance-0"
               ],
@@ -920,7 +920,7 @@ def root_event(self):
         root_event = self.make_event(data=f"{self.name} ({self.id})", event_type="SCAN", dummy=True)
         root_event._id = self.id
         root_event.scope_distance = 0
-        root_event.source = root_event
+        root_event.parent = root_event
         root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET")
         return root_event
 
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
index 14495f41a..309edcafc 100644
--- a/bbot/test/bbot_fixtures.py
+++ b/bbot/test/bbot_fixtures.py
@@ -127,47 +127,47 @@ def helpers(scan):
 @pytest.fixture
 def events(scan):
     class bbot_events:
-        localhost = scan.make_event("127.0.0.1", source=scan.root_event)
-        ipv4 = scan.make_event("8.8.8.8", source=scan.root_event)
-        netv4 = scan.make_event("8.8.8.8/30", source=scan.root_event)
-        ipv6 = scan.make_event("2001:4860:4860::8888", source=scan.root_event)
-        netv6 = scan.make_event("2001:4860:4860::8888/126", source=scan.root_event)
-        domain = scan.make_event("publicAPIs.org", source=scan.root_event)
-        subdomain = scan.make_event("api.publicAPIs.org", source=scan.root_event)
-        email = scan.make_event("bob@evilcorp.co.uk", "EMAIL_ADDRESS", source=scan.root_event)
-        open_port = scan.make_event("api.publicAPIs.org:443", source=scan.root_event)
+        localhost = scan.make_event("127.0.0.1", parent=scan.root_event)
+        ipv4 = scan.make_event("8.8.8.8", parent=scan.root_event)
+        netv4 = scan.make_event("8.8.8.8/30", parent=scan.root_event)
+        ipv6 = scan.make_event("2001:4860:4860::8888", parent=scan.root_event)
+        netv6 = scan.make_event("2001:4860:4860::8888/126", parent=scan.root_event)
+        domain = scan.make_event("publicAPIs.org", parent=scan.root_event)
+        subdomain = scan.make_event("api.publicAPIs.org", parent=scan.root_event)
+        email = scan.make_event("bob@evilcorp.co.uk", "EMAIL_ADDRESS", parent=scan.root_event)
+        open_port = scan.make_event("api.publicAPIs.org:443", parent=scan.root_event)
         protocol = scan.make_event(
-            {"host": "api.publicAPIs.org", "port": 443, "protocol": "HTTP"}, "PROTOCOL", source=scan.root_event
+            {"host": "api.publicAPIs.org", "port": 443, "protocol": "HTTP"}, "PROTOCOL", parent=scan.root_event
         )
-        ipv4_open_port = scan.make_event("8.8.8.8:443", source=scan.root_event)
-        ipv6_open_port = scan.make_event("[2001:4860:4860::8888]:443", "OPEN_TCP_PORT", source=scan.root_event)
-        url_unverified = scan.make_event("https://api.publicAPIs.org:443/hellofriend", source=scan.root_event)
-        ipv4_url_unverified = scan.make_event("https://8.8.8.8:443/hellofriend", source=scan.root_event)
-        ipv6_url_unverified = scan.make_event("https://[2001:4860:4860::8888]:443/hellofriend", source=scan.root_event)
+        ipv4_open_port = scan.make_event("8.8.8.8:443", parent=scan.root_event)
+        ipv6_open_port = scan.make_event("[2001:4860:4860::8888]:443", "OPEN_TCP_PORT", parent=scan.root_event)
+        url_unverified = scan.make_event("https://api.publicAPIs.org:443/hellofriend", parent=scan.root_event)
+        ipv4_url_unverified = scan.make_event("https://8.8.8.8:443/hellofriend", parent=scan.root_event)
+        ipv6_url_unverified = scan.make_event("https://[2001:4860:4860::8888]:443/hellofriend", parent=scan.root_event)
         url = scan.make_event(
-            "https://api.publicAPIs.org:443/hellofriend", "URL", tags=["status-200"], source=scan.root_event
+            "https://api.publicAPIs.org:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event
         )
         ipv4_url = scan.make_event(
-            "https://8.8.8.8:443/hellofriend", "URL", tags=["status-200"], source=scan.root_event
+            "https://8.8.8.8:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event
         )
         ipv6_url = scan.make_event(
-            "https://[2001:4860:4860::8888]:443/hellofriend", "URL", tags=["status-200"], source=scan.root_event
+            "https://[2001:4860:4860::8888]:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event
         )
-        url_hint = scan.make_event("https://api.publicAPIs.org:443/hello.ash", "URL_HINT", source=url)
+        url_hint = scan.make_event("https://api.publicAPIs.org:443/hello.ash", "URL_HINT", parent=url)
         vulnerability = scan.make_event(
             {"host": "evilcorp.com", "severity": "INFO", "description": "asdf"},
             "VULNERABILITY",
-            source=scan.root_event,
+            parent=scan.root_event,
         )
-        finding = scan.make_event({"host": "evilcorp.com", "description": "asdf"}, "FINDING", source=scan.root_event)
-        vhost = scan.make_event({"host": "evilcorp.com", "vhost": "www.evilcorp.com"}, "VHOST", source=scan.root_event)
-        http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event)
+        finding = scan.make_event({"host": "evilcorp.com", "description": "asdf"}, "FINDING", parent=scan.root_event)
+        vhost = scan.make_event({"host": "evilcorp.com", "vhost": "www.evilcorp.com"}, "VHOST", parent=scan.root_event)
+        http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event)
         storage_bucket = scan.make_event(
             {"name": "storage", "url": "https://storage.blob.core.windows.net"},
             "STORAGE_BUCKET",
-            source=scan.root_event,
+            parent=scan.root_event,
         )
-        emoji = scan.make_event("💩", "WHERE_IS_YOUR_GOD_NOW", source=scan.root_event)
+        emoji = scan.make_event("💩", "WHERE_IS_YOUR_GOD_NOW", parent=scan.root_event)
 
     bbot_events.all = [  # noqa: F841
         bbot_events.localhost,
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 0e8ca141b..780e58140 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -120,8 +120,8 @@ async def test_dns_resolution(bbot_scanner):
 
     # Ensure events with hosts have resolved_hosts attribute populated
     await scan._prep()
-    resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", source=scan.root_event)
-    resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", source=scan.root_event)
+    resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", parent=scan.root_event)
+    resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", parent=scan.root_event)
     dnsresolve = scan.modules["dns"]
     assert hash(resolved_hosts_event1.host) not in dnsresolve._event_cache
     assert hash(resolved_hosts_event2.host) not in dnsresolve._event_cache
@@ -206,16 +206,16 @@ async def test_wildcards(bbot_scanner):
     assert wildcard_event3.data == "github.io"
 
     # dns resolve distance
-    event_distance_0 = scan.make_event("8.8.8.8", module=scan._make_dummy_module_dns("PTR"), source=scan.root_event)
+    event_distance_0 = scan.make_event("8.8.8.8", module=scan._make_dummy_module_dns("PTR"), parent=scan.root_event)
     assert event_distance_0.dns_resolve_distance == 0
     event_distance_1 = scan.make_event(
-        "evilcorp.com", module=scan._make_dummy_module_dns("A"), source=event_distance_0
+        "evilcorp.com", module=scan._make_dummy_module_dns("A"), parent=event_distance_0
     )
     assert event_distance_1.dns_resolve_distance == 1
-    event_distance_2 = scan.make_event("1.2.3.4", module=scan._make_dummy_module_dns("PTR"), source=event_distance_1)
+    event_distance_2 = scan.make_event("1.2.3.4", module=scan._make_dummy_module_dns("PTR"), parent=event_distance_1)
     assert event_distance_2.dns_resolve_distance == 1
     event_distance_3 = scan.make_event(
-        "evilcorp.org", module=scan._make_dummy_module_dns("A"), source=event_distance_2
+        "evilcorp.org", module=scan._make_dummy_module_dns("A"), parent=event_distance_2
     )
     assert event_distance_3.dns_resolve_distance == 2
 
@@ -225,7 +225,7 @@ async def test_wildcards(bbot_scanner):
     scan2 = Scanner("asdfl.gashdgkjsadgsdf.github.io", whitelist=["github.io"], config={"dns_resolution": True})
     await scan2._prep()
     other_event = scan2.make_event(
-        "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dns"], source=scan2.root_event
+        "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dns"], parent=scan2.root_event
     )
     await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
@@ -268,7 +268,7 @@ async def test_wildcards(bbot_scanner):
     )
     await scan2._prep()
     other_event = scan2.make_event(
-        "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dns"], source=scan2.root_event
+        "lkjg.sdfgsg.jgkhajshdsadf.github.io", module=scan2.modules["dns"], parent=scan2.root_event
     )
     await scan2.ingress_module.queue_event(other_event, {})
     events = [e async for e in scan2.async_start()]
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 530969a00..461ce9b14 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -165,7 +165,7 @@ async def test_events(events, helpers):
     assert events.ipv6_url_unverified.host == ipaddress.ip_address("2001:4860:4860::8888")
     assert events.ipv6_url_unverified.port == 443
 
-    javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", source=scan.root_event)
+    javascript_event = scan.make_event("http://evilcorp.com/asdf/a.js?b=c#d", "URL_UNVERIFIED", parent=scan.root_event)
     assert "extension-js" in javascript_event.tags
     await scan.ingress_module.handle_event(javascript_event, {})
     assert "httpx-only" in javascript_event.tags
@@ -175,70 +175,70 @@ async def test_events(events, helpers):
     assert event1._scope_distance == -1
     event1.scope_distance = 0
     assert event1._scope_distance == 0
-    event2 = scan.make_event("2.3.4.5", source=event1)
+    event2 = scan.make_event("2.3.4.5", parent=event1)
     assert event2._scope_distance == 1
-    event3 = scan.make_event("3.4.5.6", source=event2)
+    event3 = scan.make_event("3.4.5.6", parent=event2)
     assert event3._scope_distance == 2
-    event4 = scan.make_event("3.4.5.6", source=event3)
+    event4 = scan.make_event("3.4.5.6", parent=event3)
     assert event4._scope_distance == 2
-    event5 = scan.make_event("4.5.6.7", source=event4)
+    event5 = scan.make_event("4.5.6.7", parent=event4)
     assert event5._scope_distance == 3
 
-    url_1 = scan.make_event("https://127.0.0.1/asdf", "URL_UNVERIFIED", source=scan.root_event)
+    url_1 = scan.make_event("https://127.0.0.1/asdf", "URL_UNVERIFIED", parent=scan.root_event)
     assert url_1.scope_distance == 1
-    url_2 = scan.make_event("https://127.0.0.1/test", "URL_UNVERIFIED", source=url_1)
+    url_2 = scan.make_event("https://127.0.0.1/test", "URL_UNVERIFIED", parent=url_1)
     assert url_2.scope_distance == 1
-    url_3 = scan.make_event("https://127.0.0.2/asdf", "URL_UNVERIFIED", source=url_1)
+    url_3 = scan.make_event("https://127.0.0.2/asdf", "URL_UNVERIFIED", parent=url_1)
     assert url_3.scope_distance == 2
 
-    org_stub_1 = scan.make_event("STUB1", "ORG_STUB", source=scan.root_event)
+    org_stub_1 = scan.make_event("STUB1", "ORG_STUB", parent=scan.root_event)
     org_stub_1.scope_distance == 1
-    org_stub_2 = scan.make_event("STUB2", "ORG_STUB", source=org_stub_1)
+    org_stub_2 = scan.make_event("STUB2", "ORG_STUB", parent=org_stub_1)
     org_stub_2.scope_distance == 2
 
     # internal event tracking
     root_event = scan.make_event("0.0.0.0", dummy=True)
-    internal_event1 = scan.make_event("1.2.3.4", source=root_event, internal=True)
+    internal_event1 = scan.make_event("1.2.3.4", parent=root_event, internal=True)
     assert internal_event1._internal == True
     assert "internal" in internal_event1.tags
 
     # tag inheritance
     for tag in ("affiliate", "mutation-1"):
-        affiliate_event = scan.make_event("1.2.3.4", source=root_event, tags=tag)
+        affiliate_event = scan.make_event("1.2.3.4", parent=root_event, tags=tag)
         assert tag in affiliate_event.tags
-        affiliate_event2 = scan.make_event("1.2.3.4:88", source=affiliate_event)
-        affiliate_event3 = scan.make_event("4.3.2.1:88", source=affiliate_event)
+        affiliate_event2 = scan.make_event("1.2.3.4:88", parent=affiliate_event)
+        affiliate_event3 = scan.make_event("4.3.2.1:88", parent=affiliate_event)
         assert tag in affiliate_event2.tags
         assert tag not in affiliate_event3.tags
 
     # updating an already-created event with make_event()
     # updating tags
-    event1 = scan.make_event("127.0.0.1", source=scan.root_event)
+    event1 = scan.make_event("127.0.0.1", parent=scan.root_event)
     updated_event = scan.make_event(event1, tags="asdf")
     assert "asdf" not in event1.tags
     assert "asdf" in updated_event.tags
-    # updating source
-    event2 = scan.make_event("127.0.0.1", source=scan.root_event)
-    updated_event = scan.make_event(event2, source=event1)
-    assert event2.source == scan.root_event
-    assert updated_event.source == event1
+    # updating parent
+    event2 = scan.make_event("127.0.0.1", parent=scan.root_event)
+    updated_event = scan.make_event(event2, parent=event1)
+    assert event2.parent == scan.root_event
+    assert updated_event.parent == event1
     # updating module
-    event3 = scan.make_event("127.0.0.1", source=scan.root_event)
+    event3 = scan.make_event("127.0.0.1", parent=scan.root_event)
     updated_event = scan.make_event(event3, internal=True)
     assert event3.internal == False
     assert updated_event.internal == True
 
     # event sorting
-    parent1 = scan.make_event("127.0.0.1", source=scan.root_event)
-    parent2 = scan.make_event("127.0.0.1", source=scan.root_event)
-    parent2_child1 = scan.make_event("127.0.0.1", source=parent2)
-    parent1_child1 = scan.make_event("127.0.0.1", source=parent1)
-    parent1_child2 = scan.make_event("127.0.0.1", source=parent1)
-    parent1_child2_child1 = scan.make_event("127.0.0.1", source=parent1_child2)
-    parent1_child2_child2 = scan.make_event("127.0.0.1", source=parent1_child2)
-    parent1_child1_child1 = scan.make_event("127.0.0.1", source=parent1_child1)
-    parent2_child2 = scan.make_event("127.0.0.1", source=parent2)
-    parent1_child2_child1_child1 = scan.make_event("127.0.0.1", source=parent1_child2_child1)
+    parent1 = scan.make_event("127.0.0.1", parent=scan.root_event)
+    parent2 = scan.make_event("127.0.0.1", parent=scan.root_event)
+    parent2_child1 = scan.make_event("127.0.0.1", parent=parent2)
+    parent1_child1 = scan.make_event("127.0.0.1", parent=parent1)
+    parent1_child2 = scan.make_event("127.0.0.1", parent=parent1)
+    parent1_child2_child1 = scan.make_event("127.0.0.1", parent=parent1_child2)
+    parent1_child2_child2 = scan.make_event("127.0.0.1", parent=parent1_child2)
+    parent1_child1_child1 = scan.make_event("127.0.0.1", parent=parent1_child1)
+    parent2_child2 = scan.make_event("127.0.0.1", parent=parent2)
+    parent1_child2_child1_child1 = scan.make_event("127.0.0.1", parent=parent1_child2_child1)
 
     sortable_events = {
         "parent1": parent1,
@@ -437,8 +437,8 @@ async def test_events(events, helpers):
     assert reconstituted_event2.host == "evilcorp.com"
     assert "127.0.0.1" in reconstituted_event2.resolved_hosts
 
-    http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", source=scan.root_event)
-    assert http_response.source_id == scan.root_event.id
+    http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event)
+    assert http_response.parent_id == scan.root_event.id
     assert http_response.data["input"] == "http://example.com:80"
     json_event = http_response.json(mode="graph")
     assert isinstance(json_event["data"], str)
@@ -446,25 +446,25 @@ async def test_events(events, helpers):
     assert isinstance(json_event["data"], dict)
     assert json_event["type"] == "HTTP_RESPONSE"
     assert json_event["host"] == "example.com"
-    assert json_event["source"] == scan.root_event.id
+    assert json_event["parent"] == scan.root_event.id
     reconstituted_event = event_from_json(json_event)
     assert isinstance(reconstituted_event.data, dict)
     assert reconstituted_event.data["input"] == "http://example.com:80"
     assert reconstituted_event.host == "example.com"
     assert reconstituted_event.type == "HTTP_RESPONSE"
-    assert reconstituted_event.source_id == scan.root_event.id
+    assert reconstituted_event.parent_id == scan.root_event.id
 
-    event_1 = scan.make_event("127.0.0.1", source=scan.root_event)
-    event_2 = scan.make_event("127.0.0.2", source=event_1)
-    event_3 = scan.make_event("127.0.0.3", source=event_2)
+    event_1 = scan.make_event("127.0.0.1", parent=scan.root_event)
+    event_2 = scan.make_event("127.0.0.2", parent=event_1)
+    event_3 = scan.make_event("127.0.0.3", parent=event_2)
     event_3._omit = True
-    event_4 = scan.make_event("127.0.0.4", source=event_3)
-    event_5 = scan.make_event("127.0.0.5", source=event_4)
-    assert event_5.get_sources() == [event_4, event_3, event_2, event_1, scan.root_event]
-    assert event_5.get_sources(omit=True) == [event_4, event_2, event_1, scan.root_event]
+    event_4 = scan.make_event("127.0.0.4", parent=event_3)
+    event_5 = scan.make_event("127.0.0.5", parent=event_4)
+    assert event_5.get_parents() == [event_4, event_3, event_2, event_1, scan.root_event]
+    assert event_5.get_parents(omit=True) == [event_4, event_2, event_1, scan.root_event]
 
     # test host backup
-    host_event = scan.make_event("asdf.evilcorp.com", "DNS_NAME", source=scan.root_event)
+    host_event = scan.make_event("asdf.evilcorp.com", "DNS_NAME", parent=scan.root_event)
     assert host_event.host_original == "asdf.evilcorp.com"
     host_event.host = "_wildcard.evilcorp.com"
     assert host_event.host == "_wildcard.evilcorp.com"
@@ -474,32 +474,32 @@ async def test_events(events, helpers):
     bucket_event = scan.make_event(
         {"name": "ASDF.s3.amazonaws.com", "url": "https://ASDF.s3.amazonaws.com"},
         "STORAGE_BUCKET",
-        source=scan.root_event,
+        parent=scan.root_event,
     )
     assert bucket_event.data["name"] == "asdf.s3.amazonaws.com"
     assert bucket_event.data["url"] == "https://asdf.s3.amazonaws.com/"
 
     # test module sequence
     module = scan._make_dummy_module("mymodule")
-    source_event_1 = scan.make_event("127.0.0.1", module=module, source=scan.root_event)
-    assert str(source_event_1.module) == "mymodule"
-    assert str(source_event_1.module_sequence) == "mymodule"
-    source_event_2 = scan.make_event("127.0.0.2", module=module, source=source_event_1)
-    assert str(source_event_2.module) == "mymodule"
-    assert str(source_event_2.module_sequence) == "mymodule"
-    source_event_3 = scan.make_event("127.0.0.3", module=module, source=source_event_2)
-    assert str(source_event_3.module) == "mymodule"
-    assert str(source_event_3.module_sequence) == "mymodule"
+    parent_event_1 = scan.make_event("127.0.0.1", module=module, parent=scan.root_event)
+    assert str(parent_event_1.module) == "mymodule"
+    assert str(parent_event_1.module_sequence) == "mymodule"
+    parent_event_2 = scan.make_event("127.0.0.2", module=module, parent=parent_event_1)
+    assert str(parent_event_2.module) == "mymodule"
+    assert str(parent_event_2.module_sequence) == "mymodule"
+    parent_event_3 = scan.make_event("127.0.0.3", module=module, parent=parent_event_2)
+    assert str(parent_event_3.module) == "mymodule"
+    assert str(parent_event_3.module_sequence) == "mymodule"
 
     module = scan._make_dummy_module("mymodule")
-    source_event_1 = scan.make_event("127.0.0.1", module=module, source=scan.root_event)
-    source_event_1._omit = True
-    assert str(source_event_1.module) == "mymodule"
-    assert str(source_event_1.module_sequence) == "mymodule"
-    source_event_2 = scan.make_event("127.0.0.2", module=module, source=source_event_1)
-    source_event_2._omit = True
-    assert str(source_event_2.module) == "mymodule"
-    assert str(source_event_2.module_sequence) == "mymodule->mymodule"
-    source_event_3 = scan.make_event("127.0.0.3", module=module, source=source_event_2)
-    assert str(source_event_3.module) == "mymodule"
-    assert str(source_event_3.module_sequence) == "mymodule->mymodule->mymodule"
+    parent_event_1 = scan.make_event("127.0.0.1", module=module, parent=scan.root_event)
+    parent_event_1._omit = True
+    assert str(parent_event_1.module) == "mymodule"
+    assert str(parent_event_1.module_sequence) == "mymodule"
+    parent_event_2 = scan.make_event("127.0.0.2", module=module, parent=parent_event_1)
+    parent_event_2._omit = True
+    assert str(parent_event_2.module) == "mymodule"
+    assert str(parent_event_2.module_sequence) == "mymodule->mymodule"
+    parent_event_3 = scan.make_event("127.0.0.3", module=module, parent=parent_event_2)
+    assert str(parent_event_3.module) == "mymodule"
+    assert str(parent_event_3.module_sequence) == "mymodule->mymodule->mymodule"
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index c9f3b49da..6ab8ce147 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -214,7 +214,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.get_file_extension("/etc/passwd") == ""
 
     assert helpers.tagify("HttP  -_Web  Title--  ") == "http-web-title"
-    tagged_event = scan.make_event("127.0.0.1", source=scan.root_event, tags=["HttP  web -__- title  "])
+    tagged_event = scan.make_event("127.0.0.1", parent=scan.root_event, tags=["HttP  web -__- title  "])
     assert "http-web-title" in tagged_event.tags
     tagged_event.remove_tag("http-web-title")
     assert "http-web-title" not in tagged_event.tags
diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
index 3f88576f9..10b79bbff 100644
--- a/bbot/test/test_step_1/test_manager_deduplication.py
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -15,7 +15,7 @@ async def setup(self):
 
         async def handle_event(self, event):
             self.events.append(event)
-            await self.emit_event(f"{self.name}.test.notreal", "DNS_NAME", source=event)
+            await self.emit_event(f"{self.name}.test.notreal", "DNS_NAME", parent=event)
 
     class EverythingModule(DefaultModule):
         _name = "everything_module"
@@ -27,7 +27,7 @@ class EverythingModule(DefaultModule):
         async def handle_event(self, event):
             self.events.append(event)
             if event.type == "DNS_NAME":
-                await self.emit_event(f"{event.data}:88", "OPEN_TCP_PORT", source=event)
+                await self.emit_event(f"{event.data}:88", "OPEN_TCP_PORT", parent=event)
 
     class NoSuppressDupes(DefaultModule):
         _name = "no_suppress_dupes"
@@ -94,20 +94,20 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert len(events) == 21
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "test.notreal"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
-    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_domain_only.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_hostport_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "test.notreal"])
-    assert 5 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "no_suppress_dupes.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_hostport_only.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "test.notreal"])
+    assert 5 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "no_suppress_dupes.test.notreal"])
 
     assert len(default_events) == 6
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
@@ -115,31 +115,31 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
-    assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+    assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
 
     assert len(all_events) == 27
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
-    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
-    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.8" and str(e.module) == "A" and e.source.data == "per_domain_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_domain_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_hostport_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "test.notreal"])
-    assert 5 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "no_suppress_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.parent.data == "no_suppress_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.parent.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.parent.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.8" and str(e.module) == "A" and e.parent.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_hostport_only.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "test.notreal"])
+    assert 5 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.parent.data == "no_suppress_dupes.test.notreal"])
 
     assert len(no_suppress_dupes) == 6
     assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
@@ -147,19 +147,19 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"])
     assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
-    assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+    assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
 
     assert len(accept_dupes) == 10
     assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
-    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
-    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"])
-    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"])
-    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_hostport_only.test.notreal"])
-    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"])
+    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "default_module.test.notreal"])
+    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "per_domain_only.test.notreal"])
+    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "per_hostport_only.test.notreal"])
+    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "test.notreal"])
     assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
-    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+    assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
 
     assert len(per_hostport_only) == 6
     assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
@@ -167,7 +167,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes"])
     assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only"])
     assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
-    assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+    assert 1 == len([e for e in per_hostport_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
 
     assert len(per_domain_only) == 1
-    assert 1 == len([e for e in per_domain_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+    assert 1 == len([e for e in per_domain_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.parent.data])
diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py
index dbca45276..84708c45d 100644
--- a/bbot/test/test_step_1/test_manager_scope_accuracy.py
+++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py
@@ -267,7 +267,7 @@ async def filter_event(self, event):
 
         async def handle_event(self, event):
             await self.emit_event(
-                {"host": str(event.host), "description": "yep", "severity": "CRITICAL"}, "VULNERABILITY", source=event
+                {"host": str(event.host), "description": "yep", "severity": "CRITICAL"}, "VULNERABILITY", parent=event
             )
 
     def custom_setup(scan):
@@ -826,12 +826,12 @@ async def test_manager_blacklist(bbot_scanner, bbot_httpserver, caplog):
 @pytest.mark.asyncio
 async def test_manager_scope_tagging(bbot_scanner):
     scan = bbot_scanner("test.notreal")
-    e1 = scan.make_event("www.test.notreal", source=scan.root_event, tags=["affiliate"])
+    e1 = scan.make_event("www.test.notreal", parent=scan.root_event, tags=["affiliate"])
     assert e1.scope_distance == 1
     assert "distance-1" in e1.tags
     assert "affiliate" in e1.tags
 
-    e2 = scan.make_event("dev.test.notreal", source=e1, tags=["affiliate"])
+    e2 = scan.make_event("dev.test.notreal", parent=e1, tags=["affiliate"])
     assert e2.scope_distance == 2
     assert "affiliate" in e2.tags
     assert "in-scope" not in e2.tags
diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 54ab1f3d9..e604e87bd 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -21,7 +21,7 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     # output module specific event filtering tests
     base_output_module_1 = BaseOutputModule(scan)
     base_output_module_1.watched_events = ["IP_ADDRESS", "URL_UNVERIFIED"]
-    localhost = scan.make_event("127.0.0.1", source=scan.root_event)
+    localhost = scan.make_event("127.0.0.1", parent=scan.root_event)
     # ip addresses should be accepted
     result, reason = base_output_module_1._event_precheck(localhost)
     assert result == True
@@ -41,12 +41,12 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     assert result == False
     assert reason == "_omit is True"
     # unwatched event types should be rejected
-    dns_name = scan.make_event("evilcorp.com", "DNS_NAME", source=scan.root_event)
+    dns_name = scan.make_event("evilcorp.com", "DNS_NAME", parent=scan.root_event)
     result, reason = base_output_module_1._event_precheck(dns_name)
     assert result == False
     assert reason == "its type is not in watched_events"
     # omitted event types matching watched events should be accepted
-    url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", source=scan.root_event)
+    url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", parent=scan.root_event)
     result, reason = base_output_module_1._event_precheck(url_unverified)
     assert result == True
     assert reason == "precheck succeeded"
@@ -54,7 +54,7 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     base_output_module_2 = BaseOutputModule(scan)
     base_output_module_2.watched_events = ["*"]
     # normal events should be accepted
-    localhost = scan.make_event("127.0.0.1", source=scan.root_event)
+    localhost = scan.make_event("127.0.0.1", parent=scan.root_event)
     result, reason = base_output_module_2._event_precheck(localhost)
     assert result == True
     assert reason == "precheck succeeded"
@@ -73,7 +73,7 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     assert result == False
     assert reason == "_omit is True"
     # omitted event types should be rejected
-    url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", source=scan.root_event)
+    url_unverified = scan.make_event("http://127.0.0.1", "URL_UNVERIFIED", parent=scan.root_event)
     result, reason = base_output_module_2._event_precheck(url_unverified)
     assert result == False
     assert reason == "its type is omitted in the config"
@@ -81,7 +81,7 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
     # common event filtering tests
     for module_class in (BaseModule, BaseOutputModule, BaseReportModule, BaseInternalModule):
         base_module = module_class(scan)
-        localhost2 = scan.make_event("127.0.0.2", source=events.subdomain)
+        localhost2 = scan.make_event("127.0.0.2", parent=events.subdomain)
         localhost2.scope_distance = 0
         # base cases
         base_module._watched_events = None
@@ -102,7 +102,7 @@ async def test_modules_basic(helpers, events, bbot_scanner, httpx_mock):
 
         # in scope only
         base_module.in_scope_only = True
-        localhost3 = scan.make_event("127.0.0.2", source=events.subdomain)
+        localhost3 = scan.make_event("127.0.0.2", parent=events.subdomain)
         valid, reason = await base_module._event_postcheck(localhost3)
         if base_module._type == "output":
             assert valid
@@ -248,11 +248,11 @@ class mod_domain_only(BaseModule):
     scan.modules["mod_domain_only"] = mod_domain_only(scan)
     scan.status = "RUNNING"
 
-    url_1 = scan.make_event("http://evilcorp.com/1", event_type="URL", source=scan.root_event, tags=["status-200"])
-    url_2 = scan.make_event("http://evilcorp.com/2", event_type="URL", source=scan.root_event, tags=["status-200"])
-    url_3 = scan.make_event("http://evilcorp.com:888/3", event_type="URL", source=scan.root_event, tags=["status-200"])
-    url_4 = scan.make_event("http://www.evilcorp.com/", event_type="URL", source=scan.root_event, tags=["status-200"])
-    url_5 = scan.make_event("http://www.evilcorp.net/", event_type="URL", source=scan.root_event, tags=["status-200"])
+    url_1 = scan.make_event("http://evilcorp.com/1", event_type="URL", parent=scan.root_event, tags=["status-200"])
+    url_2 = scan.make_event("http://evilcorp.com/2", event_type="URL", parent=scan.root_event, tags=["status-200"])
+    url_3 = scan.make_event("http://evilcorp.com:888/3", event_type="URL", parent=scan.root_event, tags=["status-200"])
+    url_4 = scan.make_event("http://www.evilcorp.com/", event_type="URL", parent=scan.root_event, tags=["status-200"])
+    url_5 = scan.make_event("http://www.evilcorp.net/", event_type="URL", parent=scan.root_event, tags=["status-200"])
 
     url_1.scope_distance = 0
     url_2.scope_distance = 0
@@ -321,11 +321,11 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_scanner,
 
         if "URL" in module.watched_events:
             url_1 = per_domain_scan.make_event(
-                "http://www.evilcorp.com/1", event_type="URL", source=per_domain_scan.root_event, tags=["status-200"]
+                "http://www.evilcorp.com/1", event_type="URL", parent=per_domain_scan.root_event, tags=["status-200"]
             )
             url_1.scope_distance = 0
             url_2 = per_domain_scan.make_event(
-                "http://mail.evilcorp.com/2", event_type="URL", source=per_domain_scan.root_event, tags=["status-200"]
+                "http://mail.evilcorp.com/2", event_type="URL", parent=per_domain_scan.root_event, tags=["status-200"]
             )
             url_2.scope_distance = 0
             valid_1, reason_1 = await module._event_postcheck(url_1)
diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py
index a2bb68bbe..41d3a6197 100644
--- a/bbot/test/test_step_1/test_scan.py
+++ b/bbot/test/test_step_1/test_scan.py
@@ -81,9 +81,9 @@ async def test_url_extension_handling(bbot_scanner):
     await scan._prep()
     assert scan.url_extension_blacklist == {"css"}
     assert scan.url_extension_httpx_only == {"js"}
-    good_event = scan.make_event("https://evilcorp.com/a.txt", "URL", tags=["status-200"], source=scan.root_event)
-    bad_event = scan.make_event("https://evilcorp.com/a.css", "URL", tags=["status-200"], source=scan.root_event)
-    httpx_event = scan.make_event("https://evilcorp.com/a.js", "URL", tags=["status-200"], source=scan.root_event)
+    good_event = scan.make_event("https://evilcorp.com/a.txt", "URL", tags=["status-200"], parent=scan.root_event)
+    bad_event = scan.make_event("https://evilcorp.com/a.css", "URL", tags=["status-200"], parent=scan.root_event)
+    httpx_event = scan.make_event("https://evilcorp.com/a.js", "URL", tags=["status-200"], parent=scan.root_event)
     assert "blacklisted" not in bad_event.tags
     assert "httpx-only" not in httpx_event.tags
     result = await scan.ingress_module.handle_event(good_event, {})
diff --git a/bbot/test/test_step_2/module_tests/test_module_cloud.py b/bbot/test/test_step_2/module_tests/test_module_cloud.py
index 1ee8df5e7..069e234b0 100644
--- a/bbot/test/test_step_2/module_tests/test_module_cloud.py
+++ b/bbot/test/test_step_2/module_tests/test_module_cloud.py
@@ -36,15 +36,15 @@ async def setup_after_prep(self, module_test):
         amazon_range = next(iter(amazon_ranges))
         amazon_address = amazon_range.broadcast_address
 
-        ip_event = scan.make_event(amazon_address, source=scan.root_event)
-        aws_event1 = scan.make_event("amazonaws.com", source=scan.root_event)
-        aws_event2 = scan.make_event("asdf.amazonaws.com", source=scan.root_event)
-        aws_event3 = scan.make_event("asdfamazonaws.com", source=scan.root_event)
-        aws_event4 = scan.make_event("test.asdf.aws", source=scan.root_event)
+        ip_event = scan.make_event(amazon_address, parent=scan.root_event)
+        aws_event1 = scan.make_event("amazonaws.com", parent=scan.root_event)
+        aws_event2 = scan.make_event("asdf.amazonaws.com", parent=scan.root_event)
+        aws_event3 = scan.make_event("asdfamazonaws.com", parent=scan.root_event)
+        aws_event4 = scan.make_event("test.asdf.aws", parent=scan.root_event)
 
-        other_event1 = scan.make_event("cname.evilcorp.com", source=scan.root_event)
-        other_event2 = scan.make_event("cname2.evilcorp.com", source=scan.root_event)
-        other_event3 = scan.make_event("cname3.evilcorp.com", source=scan.root_event)
+        other_event1 = scan.make_event("cname.evilcorp.com", parent=scan.root_event)
+        other_event2 = scan.make_event("cname2.evilcorp.com", parent=scan.root_event)
+        other_event3 = scan.make_event("cname3.evilcorp.com", parent=scan.root_event)
         other_event2._resolved_hosts = {amazon_address}
         other_event3._resolved_hosts = {"asdf.amazonaws.com"}
 
@@ -59,9 +59,9 @@ async def setup_after_prep(self, module_test):
                 t for t in event.tags if t.startswith("cloud-") or t.startswith("cdn-")
             ), f"{event} was improperly cloud-tagged"
 
-        google_event1 = scan.make_event("asdf.googleapis.com", source=scan.root_event)
-        google_event2 = scan.make_event("asdf.google", source=scan.root_event)
-        google_event3 = scan.make_event("asdf.evilcorp.com", source=scan.root_event)
+        google_event1 = scan.make_event("asdf.googleapis.com", parent=scan.root_event)
+        google_event2 = scan.make_event("asdf.google", parent=scan.root_event)
+        google_event3 = scan.make_event("asdf.evilcorp.com", parent=scan.root_event)
         google_event3._resolved_hosts = {"asdf.storage.googleapis.com"}
 
         for event in (google_event1, google_event2, google_event3):
diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py
index 73260f327..985f6e52b 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dehashed.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py
@@ -83,7 +83,7 @@ def check(self, module_test, events):
                 for e in events
                 if e.type == "USERNAME"
                 and e.data == "bob@blacklanternsecurity.com:bob@bob.com"
-                and e.source.data == "bob@blacklanternsecurity.com"
+                and e.parent.data == "bob@blacklanternsecurity.com"
             ]
         )
         assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"])
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
index d1c5e5cc9..34702b2af 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnsbrute.py
@@ -54,37 +54,37 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
         module.dedup_strategy = "lowest_parent"
 
         # test recursive brute-force event filtering
-        event = module_test.scan.make_event("blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event)
+        event = module_test.scan.make_event("blacklanternsecurity.com", "DNS_NAME", parent=module_test.scan.root_event)
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
         event = module_test.scan.make_event(
-            "www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+            "www.blacklanternsecurity.com", "DNS_NAME", parent=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
         event = module_test.scan.make_event(
-            "test.www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+            "test.www.blacklanternsecurity.com", "DNS_NAME", parent=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
         event = module_test.scan.make_event(
-            "asdf.test.www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+            "asdf.test.www.blacklanternsecurity.com", "DNS_NAME", parent=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == True
         event = module_test.scan.make_event(
-            "wat.asdf.test.www.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+            "wat.asdf.test.www.blacklanternsecurity.com", "DNS_NAME", parent=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
         assert result == False
         assert reason == f"subdomain depth of *.asdf.test.www.blacklanternsecurity.com (4) > max_depth (3)"
         event = module_test.scan.make_event(
-            "hmmm.ptr1234.blacklanternsecurity.com", "DNS_NAME", source=module_test.scan.root_event
+            "hmmm.ptr1234.blacklanternsecurity.com", "DNS_NAME", parent=module_test.scan.root_event
         )
         event.scope_distance = 0
         result, reason = await module_test.module.filter_event(event)
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py
index 4a368d908..97107451f 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_org.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py
@@ -412,7 +412,7 @@ def check(self, module_test, events):
                 and e.data["profile_name"] == "blacklanternsecurity"
                 and e.scope_distance == 1
                 and str(e.module) == "social"
-                and e.source.type == "URL_UNVERIFIED"
+                and e.parent.type == "URL_UNVERIFIED"
             ]
         )
         assert 1 == len(
@@ -424,7 +424,7 @@ def check(self, module_test, events):
                 and e.data["profile_name"] == "blacklanternsecurity"
                 and e.scope_distance == 1
                 and str(e.module) == "github_org"
-                and e.source.type == "ORG_STUB"
+                and e.parent.type == "ORG_STUB"
             ]
         )
         assert 1 == len(
diff --git a/bbot/test/test_step_2/module_tests/test_module_gowitness.py b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
index 91ecbc9f4..3d309e47a 100644
--- a/bbot/test/test_step_2/module_tests/test_module_gowitness.py
+++ b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
@@ -66,6 +66,6 @@ def check(self, module_test, events):
                 for e in events
                 if e.type == "TECHNOLOGY"
                 and e.data["url"] == "http://127.0.0.1:8888/blacklanternsecurity"
-                and e.source.type == "SOCIAL"
+                and e.parent.type == "SOCIAL"
             ]
         )
diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap.py b/bbot/test/test_step_2/module_tests/test_module_nmap.py
index 32b5495e5..b548ff024 100644
--- a/bbot/test/test_step_2/module_tests/test_module_nmap.py
+++ b/bbot/test/test_step_2/module_tests/test_module_nmap.py
@@ -8,7 +8,7 @@ class TestNmap(ModuleTestBase):
     async def setup_after_prep(self, module_test):
         # make sure our IP_RANGE / IP_ADDRESS filtering is working right
         # IPs within the target IP range should be rejected
-        ip_event_1 = module_test.scan.make_event("127.0.0.0", source=module_test.scan.root_event)
+        ip_event_1 = module_test.scan.make_event("127.0.0.0", parent=module_test.scan.root_event)
         ip_event_1.scope_distance = 0
         ip_event_1_result = await module_test.module._event_postcheck(ip_event_1)
         assert ip_event_1_result[0] == False
@@ -17,7 +17,7 @@ async def setup_after_prep(self, module_test):
             in ip_event_1_result[1]
         )
         # but ones outside should be accepted
-        ip_event_2 = module_test.scan.make_event("127.0.0.3", source=module_test.scan.root_event)
+        ip_event_2 = module_test.scan.make_event("127.0.0.3", parent=module_test.scan.root_event)
         ip_event_2.scope_distance = 0
         assert (await module_test.module._event_postcheck(ip_event_2))[0] == True
 
@@ -52,15 +52,15 @@ async def setup_after_prep(self, module_test):
         assert "127.0.0.1," in open(asset_inventory_output_file).read()
         # make sure our IP_RANGE / IP_ADDRESS filtering is working right
         # IPs within the target IP range should not be rejected because asset_inventory.use_previous=true
-        ip_event_1 = module_test.scan.make_event("127.0.0.0", source=module_test.scan.root_event)
+        ip_event_1 = module_test.scan.make_event("127.0.0.0", parent=module_test.scan.root_event)
         ip_event_1.scope_distance = 0
         assert (await module_test.module._event_postcheck(ip_event_1))[0] == True
         # but ones outside should be accepted
-        ip_event_2 = module_test.scan.make_event("127.0.0.3", source=module_test.scan.root_event)
+        ip_event_2 = module_test.scan.make_event("127.0.0.3", parent=module_test.scan.root_event)
         ip_event_2.scope_distance = 0
         assert (await module_test.module._event_postcheck(ip_event_2))[0] == True
 
-        ip_range_event = module_test.scan.make_event("127.0.0.1/31", source=module_test.scan.root_event)
+        ip_range_event = module_test.scan.make_event("127.0.0.1/31", parent=module_test.scan.root_event)
         ip_range_event.scope_distance = 0
         ip_range_filter_result = await module_test.module._event_postcheck(ip_range_event)
         assert ip_range_filter_result[0] == False
@@ -72,8 +72,8 @@ def check(self, module_test, events):
                 e
                 for e in events
                 if e.data == "127.0.0.1:8888"
-                and e.source.data == "127.0.0.1"
-                and str(e.source.module) == "asset_inventory"
+                and e.parent.data == "127.0.0.1"
+                and str(e.parent.module) == "asset_inventory"
             ]
         )
         assert not any(e.data == "127.0.0.1:8889" for e in events)
diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
index 4d7122276..e0663131f 100644
--- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
+++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
@@ -854,17 +854,17 @@ def check(self, module_test, events):
             and "Secret: [https://admin:admin@the-internet.herokuapp.com]" in e.data["description"]
         ]
         assert 3 == len(vuln_events), "Failed to find secret in events"
-        github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].source
+        github_repo_event = [e for e in vuln_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        github_workflow_event = [e for e in vuln_events if "bbot" in e.data["description"]][0].source
+        github_workflow_event = [e for e in vuln_events if "bbot" in e.data["description"]][0].parent
         file = Path(github_workflow_event.data["path"])
         assert file.is_file(), "Destination file does not exist"
-        docker_source_event = [e for e in vuln_events if e.data["host"] == "hub.docker.com"][0].source
-        file = Path(docker_source_event.data["path"])
+        docker_parent_event = [e for e in vuln_events if e.data["host"] == "hub.docker.com"][0].parent
+        file = Path(docker_parent_event.data["path"])
         assert file.is_file(), "Destination image does not exist"
 
 
@@ -881,15 +881,15 @@ def check(self, module_test, events):
             and "Secret: [https://admin:admin@internal.host.com]" in e.data["description"]
         ]
         assert 3 == len(finding_events), "Failed to find secret in events"
-        github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].source
+        github_repo_event = [e for e in finding_events if "test_keys" in e.data["description"]][0].parent
         folder = Path(github_repo_event.data["path"])
         assert folder.is_dir(), "Destination folder doesn't exist"
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        github_workflow_event = [e for e in finding_events if "bbot" in e.data["description"]][0].source
+        github_workflow_event = [e for e in finding_events if "bbot" in e.data["description"]][0].parent
         file = Path(github_workflow_event.data["path"])
         assert file.is_file(), "Destination file does not exist"
-        docker_source_event = [e for e in finding_events if e.data["host"] == "hub.docker.com"][0].source
-        file = Path(docker_source_event.data["path"])
+        docker_parent_event = [e for e in finding_events if e.data["host"] == "hub.docker.com"][0].parent
+        file = Path(docker_parent_event.data["path"])
         assert file.is_file(), "Destination image does not exist"
diff --git a/docs/dev/module_howto.md b/docs/dev/module_howto.md
index 94d8ffe60..9f9569150 100644
--- a/docs/dev/module_howto.md
+++ b/docs/dev/module_howto.md
@@ -40,7 +40,7 @@ class whois(BaseModule):
         self.hugeinfo(f"Visiting {url}")
         response = await self.helpers.request(url)
         if response is not None:
-            await self.emit_event(response.json(), "WHOIS", source=event)
+            await self.emit_event(response.json(), "WHOIS", parent=event)
 ```
 
 After saving the module, you can run it with `-m`:
diff --git a/docs/scanning/events.md b/docs/scanning/events.md
index 704141eac..241629d78 100644
--- a/docs/scanning/events.md
+++ b/docs/scanning/events.md
@@ -12,7 +12,7 @@ In addition to the obvious data (e.g. `www.evilcorp.com`), an event also contain
 
 - a `.timestamp` of when the data was discovered
 - the `.module` that discovered it
-- the `.source` event that led to its discovery
+- the `.parent` event that led to its discovery
 - its `.scope_distance` (how many hops it is from the main scope, 0 == in-scope)
 - a list of `.tags` that describe the data (`mx-record`, `http-title`, etc.)
 
@@ -28,7 +28,7 @@ These attributes allow us to construct a visual graph of events (e.g. in [Neo4j]
   "scan": "SCAN:4d786912dbc97be199da13074699c318e2067a7f",
   "timestamp": 1688526222.723366,
   "resolved_hosts": ["185.199.108.153"],
-  "source": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7",
+  "parent": "OPEN_TCP_PORT:cf7e6a937b161217eaed99f0c566eae045d094c7",
   "tags": [
     "in-scope",
     "distance-0",
diff --git a/docs/scanning/output.md b/docs/scanning/output.md
index c7d69e452..b1b0915c4 100644
--- a/docs/scanning/output.md
+++ b/docs/scanning/output.md
@@ -67,7 +67,7 @@ You will then see [events](events.md) like this:
   "scan": "SCAN:64c0e076516ae7aa6502fd99489693d0d5ec26cc",
   "timestamp": 1688518967.740472,
   "resolved_hosts": ["1.2.3.4"],
-  "source": "DNS_NAME:2da045542abbf86723f22383d04eb453e573723c",
+  "parent": "DNS_NAME:2da045542abbf86723f22383d04eb453e573723c",
   "tags": ["distance-1", "ipv4", "internal"],
   "module": "A",
   "module_sequence": "A"

From 41b8cdce6b1d2e4a7303cece035124bae17f4667 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 10:38:13 -0400
Subject: [PATCH 155/220] remove debug message

---
 bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
index f5f9976ba..370dd151a 100644
--- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
+++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
@@ -38,8 +38,6 @@ async def setup_after_prep(self, module_test):
         module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)
 
     def check(self, module_test, events):
-        for e in events:
-            self.log.critical(e)
         assert any(
             e.type == "VULNERABILITY" and "Out-of-band interaction: [Generic SSRF (GET)]" in e.data["description"]
             for e in events

From 0dd43770ab9eb05e6890bf4ef13f36ac2e55d08e Mon Sep 17 00:00:00 2001
From: BBOT Docs Autopublish <info@blacklanternsecurity.com>
Date: Thu, 16 May 2024 14:47:04 +0000
Subject: [PATCH 156/220] Refresh module docs

---
 docs/scanning/configuration.md | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index 3a02bcc9a..ac02658ea 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -236,13 +236,14 @@ Many modules accept their own configuration options. These options have the abil
 | modules.filedownload.max_filesize              | str    | Cancel download if filesize is greater than this size                                                                                                                                                                                                                                                                          | 10MB                                                                                                                                                                                                                                                                                                                                                                              |
 | modules.fingerprintx.version                   | str    | fingerprintx version                                                                                                                                                                                                                                                                                                           | 1.1.4                                                                                                                                                                                                                                                                                                                                                                             |
 | modules.gitlab.api_key                         | str    | Gitlab access token                                                                                                                                                                                                                                                                                                            |                                                                                                                                                                                                                                                                                                                                                                                   |
-| modules.gowitness.output_path                  | str    | where to save screenshots                                                                                                                                                                                                                                                                                                      |                                                                                                                                                                                                                                                                                                                                                                                   |
-| modules.gowitness.resolution_x                 | int    | screenshot resolution x                                                                                                                                                                                                                                                                                                        | 1440                                                                                                                                                                                                                                                                                                                                                                              |
-| modules.gowitness.resolution_y                 | int    | screenshot resolution y                                                                                                                                                                                                                                                                                                        | 900                                                                                                                                                                                                                                                                                                                                                                               |
+| modules.gowitness.idle_timeout                 | int    | Skip the current gowitness batch if it stalls for longer than this many seconds                                                                                                                                                                                                                                                | 1800                                                                                                                                                                                                                                                                                                                                                                              |
+| modules.gowitness.output_path                  | str    | Where to save screenshots                                                                                                                                                                                                                                                                                                      |                                                                                                                                                                                                                                                                                                                                                                                   |
+| modules.gowitness.resolution_x                 | int    | Screenshot resolution x                                                                                                                                                                                                                                                                                                        | 1440                                                                                                                                                                                                                                                                                                                                                                              |
+| modules.gowitness.resolution_y                 | int    | Screenshot resolution y                                                                                                                                                                                                                                                                                                        | 900                                                                                                                                                                                                                                                                                                                                                                               |
 | modules.gowitness.social                       | bool   | Whether to screenshot social media webpages                                                                                                                                                                                                                                                                                    | True                                                                                                                                                                                                                                                                                                                                                                              |
-| modules.gowitness.threads                      | int    | threads used to run                                                                                                                                                                                                                                                                                                            | 4                                                                                                                                                                                                                                                                                                                                                                                 |
-| modules.gowitness.timeout                      | int    | preflight check timeout                                                                                                                                                                                                                                                                                                        | 10                                                                                                                                                                                                                                                                                                                                                                                |
-| modules.gowitness.version                      | str    | gowitness version                                                                                                                                                                                                                                                                                                              | 2.4.2                                                                                                                                                                                                                                                                                                                                                                             |
+| modules.gowitness.threads                      | int    | How many gowitness threads to spawn - default is number of CPUs x 2                                                                                                                                                                                                                                                            | 4                                                                                                                                                                                                                                                                                                                                                                                 |
+| modules.gowitness.timeout                      | int    | Preflight check timeout                                                                                                                                                                                                                                                                                                        | 10                                                                                                                                                                                                                                                                                                                                                                                |
+| modules.gowitness.version                      | str    | Gowitness version                                                                                                                                                                                                                                                                                                              | 2.4.2                                                                                                                                                                                                                                                                                                                                                                             |
 | modules.httpx.in_scope_only                    | bool   | Only visit web resources that are in scope.                                                                                                                                                                                                                                                                                    | True                                                                                                                                                                                                                                                                                                                                                                              |
 | modules.httpx.max_response_size                | int    | Max response size in bytes                                                                                                                                                                                                                                                                                                     | 5242880                                                                                                                                                                                                                                                                                                                                                                           |
 | modules.httpx.probe_all_ips                    | bool   | Probe all the ips associated with same host                                                                                                                                                                                                                                                                                    | False                                                                                                                                                                                                                                                                                                                                                                             |

From dc30ea77fb4e6e30d0825d9a967a209b8c401a5e Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 11:41:04 -0400
Subject: [PATCH 157/220] fix datetime error

---
 bbot/core/event/base.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 98314095a..45ee22719 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -161,7 +161,10 @@ def __init__(
         if timestamp is not None:
             self.timestamp = timestamp
         else:
-            self.timestamp = datetime.datetime.now(datetime.UTC)
+            try:
+                self.timestamp = datetime.datetime.now(datetime.UTC)
+            except AttributeError:
+                self.timestamp = datetime.datetime.utcnow()
 
         self._tags = set()
         if tags is not None:

From 6d78969a8c53440d8b3d93619d97e1d7ad36114b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 14:59:36 -0400
Subject: [PATCH 158/220] default gowitness threads == 2x cpu count

---
 bbot/modules/gowitness.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index ca308202a..608e91406 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -15,7 +15,7 @@ class gowitness(BaseModule):
     meta = {"description": "Take screenshots of webpages"}
     options = {
         "version": "2.4.2",
-        "threads": 4,
+        "threads": 0,
         "timeout": 10,
         "resolution_x": 1440,
         "resolution_y": 900,
@@ -25,7 +25,7 @@ class gowitness(BaseModule):
     }
     options_desc = {
         "version": "Gowitness version",
-        "threads": "How many gowitness threads to spawn - default is number of CPUs x 2",
+        "threads": "How many gowitness threads to spawn (default is number of CPUs x 2)",
         "timeout": "Preflight check timeout",
         "resolution_x": "Screenshot resolution x",
         "resolution_y": "Screenshot resolution y",
@@ -90,7 +90,9 @@ async def setup(self):
         default_thread_count = min(20, num_cpus * 2)
         self.timeout = self.config.get("timeout", 10)
         self.idle_timeout = self.config.get("idle_timeout", 1800)
-        self.threads = self.config.get("threads", default_thread_count)
+        self.threads = self.config.get("threads", 0)
+        if not self.threads:
+            self.threads = default_thread_count
         self.proxy = self.scan.config.get("http_proxy", "")
         self.resolution_x = self.config.get("resolution_x")
         self.resolution_y = self.config.get("resolution_y")

From 072c0a135b5936db41968d22d4ff33232bbadb6c Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 15:05:16 -0400
Subject: [PATCH 159/220] fix github org tests

---
 bbot/modules/github_org.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py
index e5071ea8b..3aa5c34ce 100644
--- a/bbot/modules/github_org.py
+++ b/bbot/modules/github_org.py
@@ -67,7 +67,7 @@ async def handle_event(self, event):
                 org_members = await self.query_org_members(user)
                 for member in org_members:
                     event_data = {"platform": "github", "profile_name": member, "url": f"https://github.com/{member}"}
-                    member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", soparenturce=event)
+                    member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", parent=event)
                     if member_event:
                         await self.emit_event(member_event)
 

From a2ec1d52cc42d5be0f26b696bb5187a1fce646f9 Mon Sep 17 00:00:00 2001
From: BBOT Docs Autopublish <info@blacklanternsecurity.com>
Date: Thu, 16 May 2024 22:27:25 +0000
Subject: [PATCH 160/220] Refresh module docs

---
 docs/scanning/configuration.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index ac02658ea..d54fa6d50 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -241,7 +241,7 @@ Many modules accept their own configuration options. These options have the abil
 | modules.gowitness.resolution_x                 | int    | Screenshot resolution x                                                                                                                                                                                                                                                                                                        | 1440                                                                                                                                                                                                                                                                                                                                                                              |
 | modules.gowitness.resolution_y                 | int    | Screenshot resolution y                                                                                                                                                                                                                                                                                                        | 900                                                                                                                                                                                                                                                                                                                                                                               |
 | modules.gowitness.social                       | bool   | Whether to screenshot social media webpages                                                                                                                                                                                                                                                                                    | True                                                                                                                                                                                                                                                                                                                                                                              |
-| modules.gowitness.threads                      | int    | How many gowitness threads to spawn - default is number of CPUs x 2                                                                                                                                                                                                                                                            | 4                                                                                                                                                                                                                                                                                                                                                                                 |
+| modules.gowitness.threads                      | int    | How many gowitness threads to spawn (default is number of CPUs x 2)                                                                                                                                                                                                                                                            | 0                                                                                                                                                                                                                                                                                                                                                                                 |
 | modules.gowitness.timeout                      | int    | Preflight check timeout                                                                                                                                                                                                                                                                                                        | 10                                                                                                                                                                                                                                                                                                                                                                                |
 | modules.gowitness.version                      | str    | Gowitness version                                                                                                                                                                                                                                                                                                              | 2.4.2                                                                                                                                                                                                                                                                                                                                                                             |
 | modules.httpx.in_scope_only                    | bool   | Only visit web resources that are in scope.                                                                                                                                                                                                                                                                                    | True                                                                                                                                                                                                                                                                                                                                                                              |

From b9d2629ca08ce627e92b63615851a73b65838677 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 22:19:44 -0400
Subject: [PATCH 161/220] fix viewdns tests

---
 bbot/modules/viewdns.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py
index 0d996bf09..a0fbf3fa6 100644
--- a/bbot/modules/viewdns.py
+++ b/bbot/modules/viewdns.py
@@ -26,7 +26,7 @@ async def setup(self):
     async def handle_event(self, event):
         _, query = self.helpers.split_domain(event.data)
         for domain, _ in await self.query(query):
-            await self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"])
+            await self.emit_event(domain, "DNS_NAME", parent=event, tags=["affiliate"])
 
     async def query(self, query):
         results = set()

From 9975e82c6a6179c5dfffd413d48e9072078f6d3f Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 22:22:15 -0400
Subject: [PATCH 162/220] WIP discovery context

---
 bbot/core/event/base.py               |  32 ++++++
 bbot/core/helpers/names_generator.py  |   3 +-
 bbot/modules/base.py                  |  10 +-
 bbot/scanner/manager.py               |   1 +
 bbot/test/test_step_1/test_command.py |   4 +-
 bbot/test/test_step_1/test_events.py  | 149 +++++++++++++++++++++++++-
 6 files changed, 192 insertions(+), 7 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 45ee22719..ca1963587 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -111,6 +111,7 @@ def __init__(
         data,
         event_type,
         parent=None,
+        context=None,
         module=None,
         scan=None,
         scans=None,
@@ -216,6 +217,12 @@ def __init__(
         # inherit web spider distance from parent
         self.web_spider_distance = getattr(self.parent, "web_spider_distance", 0)
 
+        if (not context) and (not self._dummy):
+            raise ValidationError(f"Must specify event discovery context")
+        self._discovery_context = ""
+        if context is not None:
+            self.discovery_context = context
+
     @property
     def data(self):
         return self._data
@@ -319,6 +326,26 @@ def host_stem(self):
         else:
             return f"{self.host}"
 
+    @property
+    def discovery_context(self):
+        return self._discovery_context
+
+    @discovery_context.setter
+    def discovery_context(self, context):
+        try:
+            self._discovery_context = context.format(module=self.module, event=self)
+        except Exception as e:
+            log.warning(f"Error formatting discovery context for {self}: {e}")
+            self._discovery_context = context
+
+    @property
+    def full_discovery_context(self):
+        """
+        This event's full discovery context, including those of all its parents
+        """
+        full_event_chain = list(reversed(self.get_parents())) + [self]
+        return [e.discovery_context for e in full_event_chain if e.type != "SCAN"]
+
     @property
     def words(self):
         if self.__words is None:
@@ -1239,6 +1266,7 @@ def make_event(
     data,
     event_type=None,
     parent=None,
+    context=None,
     module=None,
     scan=None,
     scans=None,
@@ -1258,6 +1286,7 @@ def make_event(
         data (Union[str, dict, BaseEvent]): The primary data for the event or an existing event object.
         event_type (str, optional): Type of the event, e.g., 'IP_ADDRESS'. Auto-detected if not provided.
         parent (BaseEvent, optional): Parent event leading to this event's discovery.
+        context (str, optional): Description of circumstances leading to event's discovery.
         module (str, optional): Module that discovered the event.
         scan (Scan, optional): BBOT Scan object associated with the event.
         scans (List[Scan], optional): Multiple BBOT Scan objects, primarily used for unserialization.
@@ -1308,6 +1337,8 @@ def make_event(
             data.module = module
         if parent is not None:
             data.parent = parent
+        if context is not None:
+            data.context = context
         if internal == True:
             data.internal = True
         if tags:
@@ -1350,6 +1381,7 @@ def make_event(
             data,
             event_type=event_type,
             parent=parent,
+            context=context,
             module=module,
             scan=scan,
             scans=scans,
diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py
index 41e1dd0ea..baa1f13db 100644
--- a/bbot/core/helpers/names_generator.py
+++ b/bbot/core/helpers/names_generator.py
@@ -190,6 +190,7 @@
     "rambunctious",
     "rapid_unscheduled",
     "raving",
+    "reductive",
     "reckless",
     "ripped",
     "sadistic",
@@ -236,7 +237,6 @@
     "ticklish",
     "tiny",
     "tricky",
-    "tufty",
     "twitchy",
     "ugly",
     "unabated",
@@ -581,6 +581,7 @@
     "rachel",
     "radagast",
     "ralph",
+    "rambunctious",
     "randy",
     "raymond",
     "rebecca",
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 0d4985640..520f555ac 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -105,6 +105,8 @@ class BaseModule:
     batch_wait = 10
     failed_request_abort_threshold = 5
 
+    default_discovery_context = "{module} discovered {event.type}: {event.data}"
+
     _preserve_graph = False
     _stats_exclude = False
     _qsize = 1000
@@ -425,6 +427,12 @@ def make_event(self, *args, **kwargs):
             ValidationError: If the event could not be validated and raise_error is True.
         """
         raise_error = kwargs.pop("raise_error", False)
+        context = kwargs.pop("context", None)
+        if context is None:
+            kwargs["context"] = self.default_discovery_context
+        module = kwargs.pop("module", None)
+        if module is None:
+            kwargs["module"] = self
         try:
             event = self.scan.make_event(*args, **kwargs)
         except ValidationError as e:
@@ -432,8 +440,6 @@ def make_event(self, *args, **kwargs):
                 raise
             self.warning(f"{e}")
             return
-        if not event.module:
-            event.module = self
         return event
 
     async def emit_event(self, *args, **kwargs):
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 58121dd8b..8de5c1ef2 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -54,6 +54,7 @@ async def init_events(self, events=None):
                     event.parent = self.scan.root_event
                 if event.module is None:
                     event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
+                event.discovery_context = f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}"
                 self.verbose(f"Target: {event}")
                 await self.queue_event(event, {})
             await asyncio.sleep(0.1)
diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py
index 733ee9d6f..b0afdcc78 100644
--- a/bbot/test/test_step_1/test_command.py
+++ b/bbot/test/test_step_1/test_command.py
@@ -78,10 +78,10 @@ async def test_command(bbot_scanner):
 
     # test check=True
     with pytest.raises(CalledProcessError) as excinfo:
-        lines = [l async for line in scan1.helpers.run_live(["ls", "/aslkdjflasdkfsd"], check=True)]
+        lines = [line async for line in scan1.helpers.run_live(["ls", "/aslkdjflasdkfsd"], check=True)]
     assert "No such file or directory" in excinfo.value.stderr
     with pytest.raises(CalledProcessError) as excinfo:
-        lines = [l async for line in scan1.helpers.run_live(["ls", "/aslkdjflasdkfsd"], check=True, text=False)]
+        lines = [line async for line in scan1.helpers.run_live(["ls", "/aslkdjflasdkfsd"], check=True, text=False)]
     assert b"No such file or directory" in excinfo.value.stderr
     with pytest.raises(CalledProcessError) as excinfo:
         await scan1.helpers.run(["ls", "/aslkdjflasdkfsd"], check=True)
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 461ce9b14..8d027c3a6 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -3,13 +3,12 @@
 import ipaddress
 
 from ..bbot_fixtures import *
+from bbot.scanner import Scanner
 
 
 @pytest.mark.asyncio
 async def test_events(events, helpers):
 
-    from bbot.scanner import Scanner
-
     scan = Scanner()
     await scan._prep()
 
@@ -503,3 +502,149 @@ async def test_events(events, helpers):
     parent_event_3 = scan.make_event("127.0.0.3", module=module, parent=parent_event_2)
     assert str(parent_event_3.module) == "mymodule"
     assert str(parent_event_3.module_sequence) == "mymodule->mymodule->mymodule"
+
+
+@pytest.mark.asyncio
+async def test_event_discovery_context():
+
+    from bbot.modules.base import BaseModule
+
+    scan = Scanner("evilcorp.com")
+    await scan.helpers.dns._mock_dns(
+        {
+            "evilcorp.com": {"A": ["1.2.3.4"]},
+            "one.evilcorp.com": {"A": ["1.2.3.4"]},
+            "two.evilcorp.com": {"A": ["1.2.3.4"]},
+            "three.evilcorp.com": {"A": ["1.2.3.4"]},
+            "four.evilcorp.com": {"A": ["1.2.3.4"]},
+        }
+    )
+    await scan._prep()
+
+    dummy_module_1 = scan._make_dummy_module("module_1")
+    dummy_module_2 = scan._make_dummy_module("module_2")
+
+    class DummyModule(BaseModule):
+        watched_events = ["DNS_NAME"]
+        _name = "dummy_module"
+
+        async def handle_event(self, event):
+            new_event = None
+            if event.data == "evilcorp.com":
+                new_event = scan.make_event(
+                    "one.evilcorp.com",
+                    "DNS_NAME",
+                    event,
+                    context="{module} invoked forbidden magick to discover {event.type} {event.data}",
+                    module=dummy_module_1,
+                )
+            elif event.data == "one.evilcorp.com":
+                new_event = scan.make_event(
+                    "two.evilcorp.com",
+                    "DNS_NAME",
+                    event,
+                    context="{module} pledged its allegiance to cthulu and was awarded {event.type} {event.data}",
+                    module=dummy_module_1,
+                )
+            elif event.data == "two.evilcorp.com":
+                new_event = scan.make_event(
+                    "three.evilcorp.com",
+                    "DNS_NAME",
+                    event,
+                    context="{module} asked nicely and was given {event.type} {event.data}",
+                    module=dummy_module_2,
+                )
+            elif event.data == "three.evilcorp.com":
+                new_event = scan.make_event(
+                    "four.evilcorp.com",
+                    "DNS_NAME",
+                    event,
+                    context="{module} used brute force to obtain {event.type} {event.data}",
+                    module=dummy_module_2,
+                )
+            if new_event is not None:
+                await self.emit_event(new_event)
+
+    dummy_module = DummyModule(scan)
+
+    scan.modules["dummy_module"] = dummy_module
+
+    test_event = dummy_module.make_event("evilcorp.com", "DNS_NAME", parent=scan.root_event)
+    assert test_event.discovery_context == "dummy_module discovered DNS_NAME: evilcorp.com"
+
+    events = [e async for e in scan.async_start()]
+    assert len(events) == 6
+
+    assert 1 == len(
+        [
+            e
+            for e in events
+            if e.type == "DNS_NAME"
+            and e.data == "evilcorp.com"
+            and e.discovery_context == f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"
+            and e.full_discovery_context == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
+        ]
+    )
+    assert 1 == len(
+        [
+            e
+            for e in events
+            if e.type == "DNS_NAME"
+            and e.data == "one.evilcorp.com"
+            and e.discovery_context == "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com"
+            and e.full_discovery_context
+            == [
+                f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
+                "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
+            ]
+        ]
+    )
+    assert 1 == len(
+        [
+            e
+            for e in events
+            if e.type == "DNS_NAME"
+            and e.data == "two.evilcorp.com"
+            and e.discovery_context
+            == "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com"
+            and e.full_discovery_context
+            == [
+                f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
+                "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
+                "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com",
+            ]
+        ]
+    )
+    assert 1 == len(
+        [
+            e
+            for e in events
+            if e.type == "DNS_NAME"
+            and e.data == "three.evilcorp.com"
+            and e.discovery_context == "module_2 asked nicely and was given DNS_NAME three.evilcorp.com"
+            and e.full_discovery_context
+            == [
+                f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
+                "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
+                "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com",
+                "module_2 asked nicely and was given DNS_NAME three.evilcorp.com",
+            ]
+        ]
+    )
+    assert 1 == len(
+        [
+            e
+            for e in events
+            if e.type == "DNS_NAME"
+            and e.data == "four.evilcorp.com"
+            and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com"
+            and e.full_discovery_context
+            == [
+                f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
+                "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
+                "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com",
+                "module_2 asked nicely and was given DNS_NAME three.evilcorp.com",
+                "module_2 used brute force to obtain DNS_NAME four.evilcorp.com",
+            ]
+        ]
+    )

From 616c31f872db252d045e9c60617d65ae82e7cb4f Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 23:00:54 -0400
Subject: [PATCH 163/220] handle IPv6 better

---
 bbot/core/helpers/command.py |  6 +++--
 bbot/modules/portscan.py     | 45 +++++++++++++++++++++++++++++-------
 2 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/bbot/core/helpers/command.py b/bbot/core/helpers/command.py
index 06fc8a91f..40e6861cc 100644
--- a/bbot/core/helpers/command.py
+++ b/bbot/core/helpers/command.py
@@ -36,6 +36,7 @@ async def run(self, *command, check=False, text=True, **kwargs):
     # proc_tracker optionally keeps track of which processes are running under which modules
     # this allows for graceful SIGINTing of a module's processes in the case when it's killed
     proc_tracker = kwargs.pop("_proc_tracker", set())
+    log_stderr = kwargs.pop("_log_stderr", True)
     proc, _input, command = await self._spawn_proc(*command, **kwargs)
     if proc is not None:
         proc_tracker.add(proc)
@@ -56,7 +57,7 @@ async def run(self, *command, check=False, text=True, **kwargs):
             if proc.returncode:
                 if check:
                     raise CalledProcessError(proc.returncode, command, output=stdout, stderr=stderr)
-                if stderr:
+                if stderr and log_stderr:
                     command_str = " ".join(command)
                     log.warning(f"Stderr for run({command_str}):\n\t{stderr}")
 
@@ -92,6 +93,7 @@ async def run_live(self, *command, check=False, text=True, **kwargs):
     # proc_tracker optionally keeps track of which processes are running under which modules
     # this allows for graceful SIGINTing of a module's processes in the case when it's killed
     proc_tracker = kwargs.pop("_proc_tracker", set())
+    log_stderr = kwargs.pop("_log_stderr", True)
     proc, _input, command = await self._spawn_proc(*command, **kwargs)
     if proc is not None:
         proc_tracker.add(proc)
@@ -134,7 +136,7 @@ async def run_live(self, *command, check=False, text=True, **kwargs):
                 if check:
                     raise CalledProcessError(proc.returncode, command, output=stdout, stderr=stderr)
                 # surface stderr
-                if stderr:
+                if stderr and log_stderr:
                     command_str = " ".join(command)
                     log.warning(f"Stderr for run_live({command_str}):\n\t{stderr}")
         finally:
diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index 62fbfea75..20e9da31d 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -19,6 +19,9 @@ class portscan(BaseModule):
         "wait": 5,
         "ping_first": False,
         "ping_only": False,
+        "adapter_ip": "",
+        "adapter_mac": "",
+        "router_mac": "",
     }
     options_desc = {
         "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
@@ -27,6 +30,9 @@ class portscan(BaseModule):
         "wait": "Seconds to wait for replies after scan is complete",
         "ping_first": "Only portscan hosts that reply to pings",
         "ping_only": "Ping sweep only, no portscan",
+        "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
+        "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
+        "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
     }
     deps_shared = ["masscan"]
     batch_size = 1000000
@@ -37,6 +43,9 @@ async def setup(self):
         self.wait = self.config.get("wait", 10)
         self.ping_first = self.config.get("ping_first", False)
         self.ping_only = self.config.get("ping_only", False)
+        self.adapter_ip = self.config.get("adapter_ip", "")
+        self.adapter_mac = self.config.get("adapter_mac", "")
+        self.router_mac = self.config.get("router_mac", "")
         self.ports = self.config.get("ports", "")
         if self.ports:
             try:
@@ -47,6 +56,16 @@ async def setup(self):
         self.scanned_tracker = RadixTarget()
         self.prep_blacklist()
         self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
+        # check if we're set up for IPv6
+        self.ipv6_support = True
+        ipv6_result = await self.run_process(
+            ["masscan", "-p1", "--wait", "0", "-iL", self.helpers.tempfile(["::1"], pipe=False)],
+            sudo=True,
+            _log_stderr=False,
+        )
+        if ipv6_result.returncode and "failed to detect IPv6 address" in ipv6_result.stderr:
+            self.warning(f"It looks like you are not set up for IPv6. IPv6 targets will not be scanned.")
+            self.ipv6_support = False
         return True
 
     async def handle_batch(self, *events):
@@ -92,7 +111,7 @@ async def masscan(self, targets, ping=False):
             for file in (stats_file, target_file):
                 file.unlink()
 
-    def _build_masscan_command(self, target_file=None, dry_run=False, ping=False):
+    def _build_masscan_command(self, target_file=None, ping=False, dry_run=False):
         command = (
             "masscan",
             "--excludefile",
@@ -107,15 +126,22 @@ def _build_masscan_command(self, target_file=None, dry_run=False, ping=False):
         )
         if target_file is not None:
             command += ("-iL", str(target_file))
-        if ping:
-            command += ("--ping",)
+        if dry_run:
+            command += ("-p1", "--wait", "0")
         else:
-            if self.ports:
-                command += ("-p", self.ports)
+            if self.adapter_ip:
+                command += ("--adapter-ip", self.adapter_ip)
+            if self.adapter_mac:
+                command += ("--adapter-mac", self.adapter_mac)
+            if self.router_mac:
+                command += ("--router-mac", self.router_mac)
+            if ping:
+                command += ("--ping",)
             else:
-                command += ("--top-ports", str(self.top_ports))
-        if dry_run:
-            command += ("--echo",)
+                if self.ports:
+                    command += ("-p", self.ports)
+                else:
+                    command += ("--top-ports", str(self.top_ports))
         return command
 
     def make_targets(self, events):
@@ -146,6 +172,9 @@ def make_targets(self, events):
                 for h in hosts:
                     targets.add(h)
                     self.scanned_tracker.insert(h, e)
+        # remove IPv6 addresses if we're not scanning IPv6
+        if not self.ipv6_support:
+            targets = [t for t in targets if t.version != 6]
         return targets
 
     def parse_json_line(self, line):

From cce9471f63f90d83cfc16ff9f0172e0916224b12 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 16 May 2024 23:11:47 -0400
Subject: [PATCH 164/220] handle interface detection failure

---
 bbot/modules/portscan.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index 20e9da31d..291be110b 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -101,7 +101,7 @@ async def masscan(self, targets, ping=False):
 
         target_file = self.helpers.tempfile(targets, pipe=False)
         command = self._build_masscan_command(target_file, ping=ping)
-        stats_file = self.helpers.tempfile_tail(callback=self.verbose)
+        stats_file = self.helpers.tempfile_tail(callback=self.log_masscan_status)
         try:
             with open(stats_file, "w") as stats_fh:
                 async for line in self.run_process_live(command, sudo=True, stderr=stats_fh):
@@ -111,6 +111,15 @@ async def masscan(self, targets, ping=False):
             for file in (stats_file, target_file):
                 file.unlink()
 
+    def log_masscan_status(self, s):
+        if "FAIL" in s:
+            self.warning(s)
+            self.warning(
+                f'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.'
+            )
+        else:
+            self.verbose(s)
+
     def _build_masscan_command(self, target_file=None, ping=False, dry_run=False):
         command = (
             "masscan",

From bc6eef25b0729db2d796dc888b8657d6790c03a2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 04:23:15 +0000
Subject: [PATCH 165/220] Bump pytest-asyncio from 0.23.6 to 0.23.7

Bumps [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio) from 0.23.6 to 0.23.7.
- [Release notes](https://github.com/pytest-dev/pytest-asyncio/releases)
- [Commits](https://github.com/pytest-dev/pytest-asyncio/compare/v0.23.6...v0.23.7)

---
updated-dependencies:
- dependency-name: pytest-asyncio
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 2132bde85..a4ef3eabb 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1757,13 +1757,13 @@ testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygm
 
 [[package]]
 name = "pytest-asyncio"
-version = "0.23.6"
+version = "0.23.7"
 description = "Pytest support for asyncio"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-asyncio-0.23.6.tar.gz", hash = "sha256:ffe523a89c1c222598c76856e76852b787504ddb72dd5d9b6617ffa8aa2cde5f"},
-    {file = "pytest_asyncio-0.23.6-py3-none-any.whl", hash = "sha256:68516fdd1018ac57b846c9846b954f0393b26f094764a28c955eabb0536a4e8a"},
+    {file = "pytest_asyncio-0.23.7-py3-none-any.whl", hash = "sha256:009b48127fbe44518a547bddd25611551b0e43ccdbf1e67d12479f569832c20b"},
+    {file = "pytest_asyncio-0.23.7.tar.gz", hash = "sha256:5f5c72948f4c49e7db4f29f2521d4031f1c27f86e57b046126654083d4770268"},
 ]
 
 [package.dependencies]

From b7a825b4ac91b193cb6e37307bbe60b3068bd61c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 May 2024 04:23:39 +0000
Subject: [PATCH 166/220] Bump pytest from 8.1.1 to 8.2.1

Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.1 to 8.2.1.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.2.1)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 2132bde85..80ce813b6 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1410,13 +1410,13 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-
 
 [[package]]
 name = "pluggy"
-version = "1.4.0"
+version = "1.5.0"
 description = "plugin and hook calling mechanisms for python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"},
-    {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"},
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
 ]
 
 [package.extras]
@@ -1735,13 +1735,13 @@ extra = ["pygments (>=2.12)"]
 
 [[package]]
 name = "pytest"
-version = "8.1.1"
+version = "8.2.1"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"},
-    {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"},
+    {file = "pytest-8.2.1-py3-none-any.whl", hash = "sha256:faccc5d332b8c3719f40283d0d44aa5cf101cec36f88cde9ed8f2bc0538612b1"},
+    {file = "pytest-8.2.1.tar.gz", hash = "sha256:5046e5b46d8e4cac199c373041f26be56fdb81eb4e67dc11d4e10811fc3408fd"},
 ]
 
 [package.dependencies]
@@ -1749,11 +1749,11 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""}
 exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""}
 iniconfig = "*"
 packaging = "*"
-pluggy = ">=1.4,<2.0"
+pluggy = ">=1.5,<2.0"
 tomli = {version = ">=1", markers = "python_version < \"3.11\""}
 
 [package.extras]
-testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
+dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
 
 [[package]]
 name = "pytest-asyncio"

From e5e78d4bcd1608d495f4b0686cbd43b6e007c379 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 20 May 2024 08:55:40 -0400
Subject: [PATCH 167/220] test debugging

---
 bbot/test/test_step_2/module_tests/base.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 717fde226..1300c5ad3 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -121,6 +121,8 @@ async def module_test(
 
     @pytest.mark.asyncio
     async def test_module_run(self, module_test):
+        for e in module_test.events:
+            self.log.critical(f"{e} <-- {e.source}")
         self.check(module_test, module_test.events)
         module_test.log.info(f"Finished {self.name} module test")
         current_task = asyncio.current_task()

From 7a6a986ebd462fb39fe28d0eefbb7529751681d5 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 20 May 2024 13:37:35 -0400
Subject: [PATCH 168/220] support custom network adapter

---
 bbot/modules/portscan.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index 291be110b..c038cbcc5 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -19,6 +19,7 @@ class portscan(BaseModule):
         "wait": 5,
         "ping_first": False,
         "ping_only": False,
+        "adapter": "",
         "adapter_ip": "",
         "adapter_mac": "",
         "router_mac": "",
@@ -30,6 +31,7 @@ class portscan(BaseModule):
         "wait": "Seconds to wait for replies after scan is complete",
         "ping_first": "Only portscan hosts that reply to pings",
         "ping_only": "Ping sweep only, no portscan",
+        "adapter": 'Manually specify a network interface, such as "eth0" or "tun0". If not specified, the first network interface found with a default gateway will be used.',
         "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails",
         "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails",
         "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails",
@@ -43,6 +45,7 @@ async def setup(self):
         self.wait = self.config.get("wait", 10)
         self.ping_first = self.config.get("ping_first", False)
         self.ping_only = self.config.get("ping_only", False)
+        self.adapter = self.config.get("adapter", "")
         self.adapter_ip = self.config.get("adapter_ip", "")
         self.adapter_mac = self.config.get("adapter_mac", "")
         self.router_mac = self.config.get("router_mac", "")
@@ -138,6 +141,8 @@ def _build_masscan_command(self, target_file=None, ping=False, dry_run=False):
         if dry_run:
             command += ("-p1", "--wait", "0")
         else:
+            if self.adapter:
+                command += ("--adapter", self.adapter)
             if self.adapter_ip:
                 command += ("--adapter-ip", self.adapter_ip)
             if self.adapter_mac:

From 4e3202d4e369ed0c78613bdfb2b84fcc7e72e6c2 Mon Sep 17 00:00:00 2001
From: Amir <a@a.com>
Date: Mon, 20 May 2024 21:39:13 +0200
Subject: [PATCH 169/220] added author & date for modules

---
 bbot/modules/ajaxpro.py                | 6 +++++-
 bbot/modules/anubisdb.py               | 6 +++++-
 bbot/modules/azure_realm.py            | 6 +++++-
 bbot/modules/azure_tenant.py           | 6 +++++-
 bbot/modules/baddns.py                 | 6 +++++-
 bbot/modules/baddns_zone.py            | 6 +++++-
 bbot/modules/badsecrets.py             | 6 +++++-
 bbot/modules/bevigil.py                | 7 ++++++-
 bbot/modules/binaryedge.py             | 7 ++++++-
 bbot/modules/bucket_amazon.py          | 6 +++++-
 bbot/modules/bucket_azure.py           | 6 +++++-
 bbot/modules/bucket_digitalocean.py    | 6 +++++-
 bbot/modules/bucket_file_enum.py       | 4 +++-
 bbot/modules/bucket_firebase.py        | 6 +++++-
 bbot/modules/bucket_google.py          | 6 +++++-
 bbot/modules/builtwith.py              | 7 ++++++-
 bbot/modules/bypass403.py              | 6 +++++-
 bbot/modules/c99.py                    | 7 ++++++-
 bbot/modules/censys.py                 | 7 ++++++-
 bbot/modules/certspotter.py            | 6 +++++-
 bbot/modules/chaos.py                  | 7 ++++++-
 bbot/modules/code_repository.py        | 6 +++++-
 bbot/modules/columbus.py               | 6 +++++-
 bbot/modules/credshed.py               | 2 ++
 bbot/modules/crobat.py                 | 6 +++++-
 bbot/modules/crt.py                    | 6 +++++-
 bbot/modules/deadly/dastardly.py       | 6 +++++-
 bbot/modules/deadly/ffuf.py            | 6 +++++-
 bbot/modules/deadly/nuclei.py          | 6 +++++-
 bbot/modules/deadly/vhost.py           | 6 +++++-
 bbot/modules/dehashed.py               | 7 ++++++-
 bbot/modules/digitorus.py              | 6 +++++-
 bbot/modules/dnscommonsrv.py           | 6 +++++-
 bbot/modules/dnsdumpster.py            | 6 +++++-
 bbot/modules/docker_pull.py            | 6 +++++-
 bbot/modules/dockerhub.py              | 6 +++++-
 bbot/modules/dotnetnuke.py             | 6 +++++-
 bbot/modules/emailformat.py            | 6 +++++-
 bbot/modules/ffuf_shortnames.py        | 6 +++++-
 bbot/modules/filedownload.py           | 6 +++++-
 bbot/modules/fingerprintx.py           | 6 +++++-
 bbot/modules/fullhunt.py               | 7 ++++++-
 bbot/modules/generic_ssrf.py           | 6 +++++-
 bbot/modules/git.py                    | 6 +++++-
 bbot/modules/git_clone.py              | 6 +++++-
 bbot/modules/github_codesearch.py      | 7 ++++++-
 bbot/modules/github_org.py             | 6 +++++-
 bbot/modules/github_workflows.py       | 6 +++++-
 bbot/modules/gitlab.py                 | 6 +++++-
 bbot/modules/gowitness.py              | 6 +++++-
 bbot/modules/hackertarget.py           | 6 +++++-
 bbot/modules/host_header.py            | 6 +++++-
 bbot/modules/httpx.py                  | 6 +++++-
 bbot/modules/hunt.py                   | 6 +++++-
 bbot/modules/hunterio.py               | 7 ++++++-
 bbot/modules/iis_shortnames.py         | 6 +++++-
 bbot/modules/internal/aggregate.py     | 6 +++++-
 bbot/modules/internal/excavate.py      | 6 +++++-
 bbot/modules/internal/speculate.py     | 6 +++++-
 bbot/modules/internetdb.py             | 6 +++++-
 bbot/modules/ip2location.py            | 7 ++++++-
 bbot/modules/ipneighbor.py             | 6 +++++-
 bbot/modules/ipstack.py                | 7 ++++++-
 bbot/modules/leakix.py                 | 6 +++++-
 bbot/modules/masscan.py                | 6 +++++-
 bbot/modules/massdns.py                | 6 +++++-
 bbot/modules/myssl.py                  | 6 +++++-
 bbot/modules/newsletters.py            | 6 +++++-
 bbot/modules/nmap.py                   | 6 +++++-
 bbot/modules/ntlm.py                   | 6 +++++-
 bbot/modules/oauth.py                  | 6 +++++-
 bbot/modules/otx.py                    | 6 +++++-
 bbot/modules/output/asset_inventory.py | 6 +++++-
 bbot/modules/output/csv.py             | 6 +++++-
 bbot/modules/output/discord.py         | 6 +++++-
 bbot/modules/output/emails.py          | 6 +++++-
 bbot/modules/output/http.py            | 6 +++++-
 bbot/modules/output/human.py           | 6 +++++-
 bbot/modules/output/json.py            | 6 +++++-
 bbot/modules/output/neo4j.py           | 6 +++++-
 bbot/modules/output/python.py          | 6 +++++-
 bbot/modules/output/slack.py           | 6 +++++-
 bbot/modules/output/splunk.py          | 6 +++++-
 bbot/modules/output/subdomains.py      | 6 +++++-
 bbot/modules/output/teams.py           | 6 +++++-
 bbot/modules/output/web_report.py      | 6 +++++-
 bbot/modules/output/websocket.py       | 6 +++++-
 bbot/modules/paramminer_cookies.py     | 2 ++
 bbot/modules/paramminer_getparams.py   | 6 +++++-
 bbot/modules/paramminer_headers.py     | 6 +++++-
 bbot/modules/passivetotal.py           | 7 ++++++-
 bbot/modules/pgp.py                    | 6 +++++-
 bbot/modules/postman.py                | 6 +++++-
 bbot/modules/rapiddns.py               | 6 +++++-
 bbot/modules/report/affiliates.py      | 6 +++++-
 bbot/modules/report/asn.py             | 6 +++++-
 bbot/modules/riddler.py                | 6 +++++-
 bbot/modules/robots.py                 | 6 +++++-
 bbot/modules/secretsdb.py              | 6 +++++-
 bbot/modules/securitytrails.py         | 7 ++++++-
 bbot/modules/shodan_dns.py             | 7 ++++++-
 bbot/modules/sitedossier.py            | 6 +++++-
 bbot/modules/skymem.py                 | 6 +++++-
 bbot/modules/smuggler.py               | 6 +++++-
 bbot/modules/social.py                 | 6 +++++-
 bbot/modules/sslcert.py                | 2 ++
 bbot/modules/subdomaincenter.py        | 6 +++++-
 bbot/modules/sublist3r.py              | 2 ++
 bbot/modules/telerik.py                | 6 +++++-
 bbot/modules/threatminer.py            | 2 ++
 bbot/modules/trufflehog.py             | 6 +++++-
 bbot/modules/url_manipulation.py       | 6 +++++-
 bbot/modules/urlscan.py                | 2 ++
 bbot/modules/viewdns.py                | 2 ++
 bbot/modules/virustotal.py             | 7 ++++++-
 bbot/modules/wafw00f.py                | 6 +++++-
 bbot/modules/wappalyzer.py             | 2 ++
 bbot/modules/wayback.py                | 2 ++
 bbot/modules/zoomeye.py                | 7 ++++++-
 119 files changed, 583 insertions(+), 110 deletions(-)

diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py
index 46d475cca..019a374bf 100644
--- a/bbot/modules/ajaxpro.py
+++ b/bbot/modules/ajaxpro.py
@@ -11,7 +11,11 @@ class ajaxpro(BaseModule):
     watched_events = ["HTTP_RESPONSE", "URL"]
     produced_events = ["VULNERABILITY", "FINDING"]
     flags = ["active", "safe", "web-thorough"]
-    meta = {"description": "Check for potentially vulnerable Ajaxpro instances"}
+    meta = {
+        "description": "Check for potentially vulnerable Ajaxpro instances",
+        "created_date": "2024-01-18",
+        "author": "@liquidsec"
+        }
 
     async def handle_event(self, event):
         if event.type == "URL":
diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py
index 9864e3c6d..fa9c21257 100644
--- a/bbot/modules/anubisdb.py
+++ b/bbot/modules/anubisdb.py
@@ -5,7 +5,11 @@ class anubisdb(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query jldc.me's database for subdomains"}
+    meta = {
+        "description": "Query jldc.me's database for subdomains",
+        "created_date": "2022-10-04",
+        "author": "@TheTechromancer"
+        }
     options = {"limit": 1000}
     options_desc = {
         "limit": "Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API)"
diff --git a/bbot/modules/azure_realm.py b/bbot/modules/azure_realm.py
index 33772921e..e65341fa2 100644
--- a/bbot/modules/azure_realm.py
+++ b/bbot/modules/azure_realm.py
@@ -5,7 +5,11 @@ class azure_realm(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "web-thorough", "passive", "safe"]
-    meta = {"description": 'Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm'}
+    meta = {
+        "description": 'Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm',
+        "created_date": "2023-07-12",
+        "author": "@TheTechromancer"
+        }
 
     async def setup(self):
         self.processed = set()
diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index 909acbe20..bc94480bc 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -8,7 +8,11 @@ class azure_tenant(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["affiliates", "subdomain-enum", "cloud-enum", "passive", "safe"]
-    meta = {"description": "Query Azure for tenant sister domains"}
+    meta = {
+        "description": "Query Azure for tenant sister domains",
+        "created_date": "2024-07-04",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://autodiscover-s.outlook.com"
     in_scope_only = True
diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py
index 9abfebc84..987cc0827 100644
--- a/bbot/modules/baddns.py
+++ b/bbot/modules/baddns.py
@@ -13,7 +13,11 @@ class baddns(BaseModule):
     watched_events = ["DNS_NAME", "DNS_NAME_UNRESOLVED"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["active", "safe", "web-basic", "baddns", "cloud-enum", "subdomain-hijack"]
-    meta = {"description": "Check hosts for domain/subdomain takeovers"}
+    meta = {
+        "description": "Check hosts for domain/subdomain takeovers",
+        "created_date": "2024-01-18",
+        "author": "@liquidsec"
+        }
     options = {"custom_nameservers": [], "only_high_confidence": False}
     options_desc = {
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py
index a42fe2e21..7b99af1ae 100644
--- a/bbot/modules/baddns_zone.py
+++ b/bbot/modules/baddns_zone.py
@@ -11,7 +11,11 @@ class baddns_zone(baddns_module):
     watched_events = ["DNS_NAME"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["active", "safe", "subdomain-enum", "baddns", "cloud-enum"]
-    meta = {"description": "Check hosts for DNS zone transfers and NSEC walks"}
+    meta = {
+        "description": "Check hosts for DNS zone transfers and NSEC walks",
+        "created_date": "2024-01-29",
+        "author": "@liquidsec"
+        }
     options = {"custom_nameservers": [], "only_high_confidence": False}
     options_desc = {
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index 7fde4a8e3..e8e4074d4 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -9,7 +9,11 @@ class badsecrets(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING", "VULNERABILITY", "TECHNOLOGY"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {"description": "Library for detecting known or weak secrets across many web frameworks"}
+    meta = {
+        "description": "Library for detecting known or weak secrets across many web frameworks",
+        "created_date": "2022-11-19",
+        "author": "@liquidsec"
+        }
     deps_pip = ["badsecrets~=0.4.490"]
 
     @property
diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py
index 435ceae08..85a42c8b0 100644
--- a/bbot/modules/bevigil.py
+++ b/bbot/modules/bevigil.py
@@ -9,7 +9,12 @@ class bevigil(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME", "URL_UNVERIFIED"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Retrieve OSINT data from mobile applications using BeVigil", "auth_required": True}
+    meta = {
+        "description": "Retrieve OSINT data from mobile applications using BeVigil",
+        "created_date": "2022-10-26",
+        "author": "@alt-glitch",
+        "auth_required": True
+        }
     options = {"api_key": "", "urls": False}
     options_desc = {"api_key": "BeVigil OSINT API Key", "urls": "Emit URLs in addition to DNS_NAMEs"}
 
diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py
index 64970c861..f65aaf6c6 100644
--- a/bbot/modules/binaryedge.py
+++ b/bbot/modules/binaryedge.py
@@ -5,7 +5,12 @@ class binaryedge(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the BinaryEdge API", "auth_required": True}
+    meta = {
+        "description": "Query the BinaryEdge API",
+        "created_date": "2024-08-18",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": "", "max_records": 1000}
     options_desc = {
         "api_key": "BinaryEdge API key",
diff --git a/bbot/modules/bucket_amazon.py b/bbot/modules/bucket_amazon.py
index ddb6d05f3..93289378c 100644
--- a/bbot/modules/bucket_amazon.py
+++ b/bbot/modules/bucket_amazon.py
@@ -5,7 +5,11 @@ class bucket_amazon(bucket_template):
     watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
     produced_events = ["STORAGE_BUCKET", "FINDING"]
     flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
-    meta = {"description": "Check for S3 buckets related to target"}
+    meta = {
+        "description": "Check for S3 buckets related to target",
+        "created_date": "2022-11-04",
+        "author": "@TheTechromancer"
+        }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_azure.py b/bbot/modules/bucket_azure.py
index 81e441b5f..0261f6ccb 100644
--- a/bbot/modules/bucket_azure.py
+++ b/bbot/modules/bucket_azure.py
@@ -5,7 +5,11 @@ class bucket_azure(bucket_template):
     watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
     produced_events = ["STORAGE_BUCKET", "FINDING"]
     flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
-    meta = {"description": "Check for Azure storage blobs related to target"}
+    meta = {
+        "description": "Check for Azure storage blobs related to target",
+        "created_date": "2022-11-04",
+        "author": "@TheTechromancer"
+        }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_digitalocean.py b/bbot/modules/bucket_digitalocean.py
index c467fde6e..85bb7c6f3 100644
--- a/bbot/modules/bucket_digitalocean.py
+++ b/bbot/modules/bucket_digitalocean.py
@@ -5,7 +5,11 @@ class bucket_digitalocean(bucket_template):
     watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
     produced_events = ["STORAGE_BUCKET", "FINDING"]
     flags = ["active", "safe", "slow", "cloud-enum", "web-thorough"]
-    meta = {"description": "Check for DigitalOcean spaces related to target"}
+    meta = {
+        "description": "Check for DigitalOcean spaces related to target",
+        "created_date": "2022-11-08",
+        "author": "@TheTechromancer"
+        }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py
index facaa021e..265b1bcff 100644
--- a/bbot/modules/bucket_file_enum.py
+++ b/bbot/modules/bucket_file_enum.py
@@ -10,7 +10,9 @@ class bucket_file_enum(BaseModule):
     watched_events = ["STORAGE_BUCKET"]
     produced_events = ["URL_UNVERIFIED"]
     meta = {
-        "description": "Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS"
+        "description": "Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS",
+        "created_date": "2023-11-14",
+        "author": "@TheTechromancer"
     }
     flags = ["passive", "safe", "cloud-enum"]
     options = {
diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py
index 9c883bc9c..db619149f 100644
--- a/bbot/modules/bucket_firebase.py
+++ b/bbot/modules/bucket_firebase.py
@@ -5,7 +5,11 @@ class bucket_firebase(bucket_template):
     watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
     produced_events = ["STORAGE_BUCKET", "FINDING"]
     flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
-    meta = {"description": "Check for open Firebase databases related to target"}
+    meta = {
+        "description": "Check for open Firebase databases related to target",
+        "created_date": "2023-03-20",
+        "author": "@TheTechromancer"
+        }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_google.py b/bbot/modules/bucket_google.py
index 0bd22f0f1..3964072f4 100644
--- a/bbot/modules/bucket_google.py
+++ b/bbot/modules/bucket_google.py
@@ -9,7 +9,11 @@ class bucket_google(bucket_template):
     watched_events = ["DNS_NAME", "STORAGE_BUCKET"]
     produced_events = ["STORAGE_BUCKET", "FINDING"]
     flags = ["active", "safe", "cloud-enum", "web-basic", "web-thorough"]
-    meta = {"description": "Check for Google object storage related to target"}
+    meta = {
+        "description": "Check for Google object storage related to target",
+        "created_date": "2022-11-04",
+        "author": "@TheTechromancer"
+        }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py
index 0b5793657..bc091122a 100644
--- a/bbot/modules/builtwith.py
+++ b/bbot/modules/builtwith.py
@@ -17,7 +17,12 @@ class builtwith(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["affiliates", "subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query Builtwith.com for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query Builtwith.com for subdomains",
+        "created_date": "2022-08-23",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": "", "redirects": True}
     options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"}
     base_url = "https://api.builtwith.com"
diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index c58463401..0a8de646c 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -79,7 +79,11 @@ class bypass403(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {"description": "Check 403 pages for common bypasses"}
+    meta = {
+        "description": "Check 403 pages for common bypasses",
+        "created_date": "2022-07-05",
+        "author": "@liquidsec"
+        }
     in_scope_only = True
 
     async def do_checks(self, compare_helper, event, collapse_threshold):
diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py
index 8e05a1c4b..128c0a371 100644
--- a/bbot/modules/c99.py
+++ b/bbot/modules/c99.py
@@ -5,7 +5,12 @@ class c99(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the C99 API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query the C99 API for subdomains",
+        "created_date": "2022-07-08",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "c99.nl API key"}
 
diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py
index b8609adf8..1580a0af8 100644
--- a/bbot/modules/censys.py
+++ b/bbot/modules/censys.py
@@ -9,7 +9,12 @@ class censys(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the Censys API", "auth_required": True}
+    meta = {
+        "description": "Query the Censys API",
+        "created_date": "2022-08-04",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_id": "", "api_secret": "", "max_pages": 5}
     options_desc = {
         "api_id": "Censys.io API ID",
diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py
index 4441b9d98..caafe1898 100644
--- a/bbot/modules/certspotter.py
+++ b/bbot/modules/certspotter.py
@@ -5,7 +5,11 @@ class certspotter(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query Certspotter's API for subdomains"}
+    meta = {
+        "description": "Query Certspotter's API for subdomains",
+        "created_date": "2022-07-28",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://api.certspotter.com/v1"
 
diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py
index 3eb763573..0237a8a20 100644
--- a/bbot/modules/chaos.py
+++ b/bbot/modules/chaos.py
@@ -5,7 +5,12 @@ class chaos(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query ProjectDiscovery's Chaos API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query ProjectDiscovery's Chaos API for subdomains",
+        "created_date": "2022-08-14",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "Chaos API key"}
 
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
index 5c55e41c3..4f6ed6049 100644
--- a/bbot/modules/code_repository.py
+++ b/bbot/modules/code_repository.py
@@ -5,7 +5,11 @@
 class code_repository(BaseModule):
     watched_events = ["URL_UNVERIFIED"]
     produced_events = ["CODE_REPOSITORY"]
-    meta = {"description": "Look for code repository links in webpages"}
+    meta = {
+        "description": "Look for code repository links in webpages",
+        "created_date": "2024-05-15",
+        "author": "@domwhewell-sage"
+        }
     flags = ["passive", "safe", "repo-enum"]
 
     # platform name : (regex, case_sensitive)
diff --git a/bbot/modules/columbus.py b/bbot/modules/columbus.py
index 2e8901359..b4d5370d6 100644
--- a/bbot/modules/columbus.py
+++ b/bbot/modules/columbus.py
@@ -5,7 +5,11 @@ class columbus(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query the Columbus Project API for subdomains"}
+    meta = {
+        "description": "Query the Columbus Project API for subdomains",
+        "created_date": "2023-06-01",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://columbus.elmasy.com/api/lookup"
 
diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py
index 382644007..eee229e76 100644
--- a/bbot/modules/credshed.py
+++ b/bbot/modules/credshed.py
@@ -9,6 +9,8 @@ class credshed(BaseModule):
     flags = ["passive", "safe"]
     meta = {
         "description": "Send queries to your own credshed server to check for known credentials of your targets",
+        "created_date": "2023-10-12",
+        "author": "@SpamFaux",
         "auth_required": True,
     }
     options = {"username": "", "password": "", "credshed_url": ""}
diff --git a/bbot/modules/crobat.py b/bbot/modules/crobat.py
index 7ece44fdb..b5a26c705 100644
--- a/bbot/modules/crobat.py
+++ b/bbot/modules/crobat.py
@@ -6,5 +6,9 @@ class crobat(subdomain_enum):
     produced_events = ["DNS_NAME"]
     # tag "subdomain-enum" removed 2023-02-24 because API is offline
     flags = ["passive", "safe"]
-    meta = {"description": "Query Project Crobat for subdomains"}
+    meta = {
+        "description": "Query Project Crobat for subdomains",
+        "created_date": "2022-06-03",
+        "author": "@j3tj3rk"
+        }
     base_url = "https://sonar.omnisint.io"
diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py
index 9773f72d4..a2d995a58 100644
--- a/bbot/modules/crt.py
+++ b/bbot/modules/crt.py
@@ -5,7 +5,11 @@ class crt(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query crt.sh (certificate transparency) for subdomains"}
+    meta = {
+        "description": "Query crt.sh (certificate transparency) for subdomains",
+        "created_date": "2022-05-13",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://crt.sh"
     reject_wildcards = False
diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py
index bfba6c636..d78f1d257 100644
--- a/bbot/modules/deadly/dastardly.py
+++ b/bbot/modules/deadly/dastardly.py
@@ -6,7 +6,11 @@ class dastardly(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["active", "aggressive", "slow", "web-thorough"]
-    meta = {"description": "Lightweight web application security scanner"}
+    meta = {
+        "description": "Lightweight web application security scanner",
+        "created_date": "2023-12-11",
+        "author": "@domwhewell-sage"
+        }
 
     deps_pip = ["lxml~=4.9.2"]
     deps_ansible = [
diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py
index 8382f1e66..f08b5d507 100644
--- a/bbot/modules/deadly/ffuf.py
+++ b/bbot/modules/deadly/ffuf.py
@@ -11,7 +11,11 @@ class ffuf(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["aggressive", "active"]
-    meta = {"description": "A fast web fuzzer written in Go"}
+    meta = {
+        "description": "A fast web fuzzer written in Go",
+        "created_date": "2022-04-10",
+        "author": "@pmueller"
+        }
 
     options = {
         "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt",
diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py
index 9234f7494..14e4c1692 100644
--- a/bbot/modules/deadly/nuclei.py
+++ b/bbot/modules/deadly/nuclei.py
@@ -8,7 +8,11 @@ class nuclei(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING", "VULNERABILITY", "TECHNOLOGY"]
     flags = ["active", "aggressive"]
-    meta = {"description": "Fast and customisable vulnerability scanner"}
+    meta = {
+        "description": "Fast and customisable vulnerability scanner",
+        "created_date": "2022-03-12",
+        "author": "@TheTechromancer"
+        }
 
     options = {
         "version": "3.2.0",
diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py
index e2908dbbe..ba436ee4e 100644
--- a/bbot/modules/deadly/vhost.py
+++ b/bbot/modules/deadly/vhost.py
@@ -8,7 +8,11 @@ class vhost(ffuf):
     watched_events = ["URL"]
     produced_events = ["VHOST", "DNS_NAME"]
     flags = ["active", "aggressive", "slow"]
-    meta = {"description": "Fuzz for virtual hosts"}
+    meta = {
+        "description": "Fuzz for virtual hosts",
+        "created_date": "2022-05-02",
+        "author": "@liquidsec"
+        }
 
     special_vhost_list = ["127.0.0.1", "localhost", "host.docker.internal"]
     options = {
diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py
index c1a35c419..5a7016103 100644
--- a/bbot/modules/dehashed.py
+++ b/bbot/modules/dehashed.py
@@ -7,7 +7,12 @@ class dehashed(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"]
     flags = ["passive", "safe", "email-enum"]
-    meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True}
+    meta = {
+        "description": "Execute queries against dehashed.com for exposed credentials",
+        "created_date": "2023-10-12",
+        "author": "@SpamFaux",
+        "auth_required": True
+        }
     options = {"username": "", "api_key": ""}
     options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"}
     target_only = True
diff --git a/bbot/modules/digitorus.py b/bbot/modules/digitorus.py
index 0da487744..b51c0e49f 100644
--- a/bbot/modules/digitorus.py
+++ b/bbot/modules/digitorus.py
@@ -7,7 +7,11 @@ class digitorus(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query certificatedetails.com for subdomains"}
+    meta = {
+        "description": "Query certificatedetails.com for subdomains",
+        "created_date": "2023-07-25",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://certificatedetails.com"
 
diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py
index eef8e2d8c..800015a71 100644
--- a/bbot/modules/dnscommonsrv.py
+++ b/bbot/modules/dnscommonsrv.py
@@ -153,7 +153,11 @@ class dnscommonsrv(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Check for common SRV records"}
+    meta = {
+        "description": "Check for common SRV records",
+        "created_date": "2022-05-15",
+        "author": "@TheTechromancer"
+        }
     options = {"top": 50, "max_event_handlers": 10}
     options_desc = {
         "top": "How many of the top SRV records to check",
diff --git a/bbot/modules/dnsdumpster.py b/bbot/modules/dnsdumpster.py
index 06986f216..40e2dd0de 100644
--- a/bbot/modules/dnsdumpster.py
+++ b/bbot/modules/dnsdumpster.py
@@ -7,7 +7,11 @@ class dnsdumpster(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query dnsdumpster for subdomains"}
+    meta = {
+        "description": "Query dnsdumpster for subdomains",
+        "created_date": "2022-03-12",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://dnsdumpster.com"
 
diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py
index 86c7a07a7..80071d2ca 100644
--- a/bbot/modules/docker_pull.py
+++ b/bbot/modules/docker_pull.py
@@ -9,7 +9,11 @@ class docker_pull(BaseModule):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
     flags = ["passive", "safe", "slow"]
-    meta = {"description": "Download images from a docker repository"}
+    meta = {
+        "description": "Download images from a docker repository",
+        "created_date": "2024-03-24",
+        "author": "@domwhewell-sage"
+        }
     options = {"all_tags": False, "output_folder": ""}
     options_desc = {
         "all_tags": "Download all tags from each registry (Default False)",
diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py
index 5829e57d9..e2940636a 100644
--- a/bbot/modules/dockerhub.py
+++ b/bbot/modules/dockerhub.py
@@ -5,7 +5,11 @@ class dockerhub(BaseModule):
     watched_events = ["SOCIAL", "ORG_STUB"]
     produced_events = ["SOCIAL", "CODE_REPOSITORY", "URL_UNVERIFIED"]
     flags = ["passive", "safe"]
-    meta = {"description": "Search for docker repositories of discovered orgs/usernames"}
+    meta = {
+        "description": "Search for docker repositories of discovered orgs/usernames",
+        "created_date": "2024-03-12",
+        "author": "@domwhewell-sage"
+        }
 
     site_url = "https://hub.docker.com"
     api_url = f"{site_url}/v2"
diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index ea3d7b920..04588cd0c 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -20,7 +20,11 @@ class dotnetnuke(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["VULNERABILITY", "TECHNOLOGY"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {"description": "Scan for critical DotNetNuke (DNN) vulnerabilities"}
+    meta = {
+        "description": "Scan for critical DotNetNuke (DNN) vulnerabilities",
+        "created_date": "2023-11-21",
+        "author": "@liquidsec"
+        }
 
     async def setup(self):
         self.event_dict = {}
diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py
index 000c3d5cf..92d98bb29 100644
--- a/bbot/modules/emailformat.py
+++ b/bbot/modules/emailformat.py
@@ -5,7 +5,11 @@ class emailformat(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["EMAIL_ADDRESS"]
     flags = ["passive", "email-enum", "safe"]
-    meta = {"description": "Query email-format.com for email addresses"}
+    meta = {
+        "description": "Query email-format.com for email addresses",
+        "created_date": "2022-07-11",
+        "author": "@TheTechromancer"
+        }
     in_scope_only = False
     per_domain_only = True
 
diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py
index 562ae681f..3a3d91392 100644
--- a/bbot/modules/ffuf_shortnames.py
+++ b/bbot/modules/ffuf_shortnames.py
@@ -33,7 +33,11 @@ class ffuf_shortnames(ffuf):
     watched_events = ["URL_HINT"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["aggressive", "active", "iis-shortnames", "web-thorough"]
-    meta = {"description": "Use ffuf in combination IIS shortnames"}
+    meta = {
+        "description": "Use ffuf in combination IIS shortnames",
+        "created_date": "2022-07-05",
+        "author": "@liquidsec"
+        }
 
     options = {
         "wordlist": "",  # default is defined within setup function
diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py
index e82463a48..706fd8c44 100644
--- a/bbot/modules/filedownload.py
+++ b/bbot/modules/filedownload.py
@@ -15,7 +15,11 @@ class filedownload(BaseModule):
     watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"]
     produced_events = []
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {"description": "Download common filetypes such as PDF, DOCX, PPTX, etc."}
+    meta = {
+        "description": "Download common filetypes such as PDF, DOCX, PPTX, etc.",
+        "created_date": "2023-10-11",
+        "author": "@TheTechromancer"
+        }
     options = {
         "extensions": [
             "bak",  #  Backup File
diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py
index 41efa75b6..ce2c98303 100644
--- a/bbot/modules/fingerprintx.py
+++ b/bbot/modules/fingerprintx.py
@@ -7,7 +7,11 @@ class fingerprintx(BaseModule):
     watched_events = ["OPEN_TCP_PORT"]
     produced_events = ["PROTOCOL"]
     flags = ["active", "safe", "service-enum", "slow"]
-    meta = {"description": "Fingerprint exposed services like RDP, SSH, MySQL, etc."}
+    meta = {
+        "description": "Fingerprint exposed services like RDP, SSH, MySQL, etc.",
+        "created_date": "2023-01-30",
+        "author": "@TheTechromancer"
+        }
     options = {"version": "1.1.4"}
     options_desc = {"version": "fingerprintx version"}
     _batch_size = 10
diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py
index 1485dc6b5..08e870bcf 100644
--- a/bbot/modules/fullhunt.py
+++ b/bbot/modules/fullhunt.py
@@ -5,7 +5,12 @@ class fullhunt(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the fullhunt.io API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query the fullhunt.io API for subdomains",
+        "created_date": "2022-08-24",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "FullHunt API Key"}
 
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index 9d75f4a9e..936f09b29 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -152,7 +152,11 @@ class generic_ssrf(BaseModule):
     watched_events = ["URL"]
     produced_events = ["VULNERABILITY"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {"description": "Check for generic SSRFs"}
+    meta = {
+        "description": "Check for generic SSRFs",
+        "created_date": "2022-07-30",
+        "author": "@liquidsec"
+        }
     in_scope_only = True
 
     deps_apt = ["curl"]
diff --git a/bbot/modules/git.py b/bbot/modules/git.py
index 0b19f7e6a..7bedeb650 100644
--- a/bbot/modules/git.py
+++ b/bbot/modules/git.py
@@ -7,7 +7,11 @@ class git(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {"description": "Check for exposed .git repositories"}
+    meta = {
+        "description": "Check for exposed .git repositories",
+        "created_date": "2023-05-30",
+        "author": "@TheTechromancer"
+        }
 
     in_scope_only = True
 
diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py
index 2778efdde..bf6ccc1ff 100644
--- a/bbot/modules/git_clone.py
+++ b/bbot/modules/git_clone.py
@@ -6,7 +6,11 @@ class git_clone(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
     flags = ["passive", "safe", "slow"]
-    meta = {"description": "Clone code github repositories"}
+    meta = {
+        "description": "Clone code github repositories",
+        "created_date": "2024-03-08",
+        "author": "@domwhewell-sage"
+        }
     options = {"api_key": "", "output_folder": ""}
     options_desc = {"api_key": "Github token", "output_folder": "Folder to clone repositories to"}
 
diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py
index 6a3ea57a5..3952f6758 100644
--- a/bbot/modules/github_codesearch.py
+++ b/bbot/modules/github_codesearch.py
@@ -5,7 +5,12 @@ class github_codesearch(github):
     watched_events = ["DNS_NAME"]
     produced_events = ["CODE_REPOSITORY", "URL_UNVERIFIED"]
     flags = ["passive", "subdomain-enum", "safe"]
-    meta = {"description": "Query Github's API for code containing the target domain name", "auth_required": True}
+    meta = {
+        "description": "Query Github's API for code containing the target domain name",
+        "created_date": "2023-12-14",
+        "author": "@domwhewell-sage",
+        "auth_required": True
+        }
     options = {"api_key": "", "limit": 100}
     options_desc = {"api_key": "Github token", "limit": "Limit code search to this many results"}
 
diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py
index 93cefa9ee..4dce3a7ca 100644
--- a/bbot/modules/github_org.py
+++ b/bbot/modules/github_org.py
@@ -5,7 +5,11 @@ class github_org(github):
     watched_events = ["ORG_STUB", "SOCIAL"]
     produced_events = ["CODE_REPOSITORY"]
     flags = ["passive", "subdomain-enum", "safe"]
-    meta = {"description": "Query Github's API for organization and member repositories"}
+    meta = {
+        "description": "Query Github's API for organization and member repositories",
+        "created_date": "2023-12-14",
+        "author": "@domwhewell-sage"
+        }
     options = {"api_key": "", "include_members": True, "include_member_repos": False}
     options_desc = {
         "api_key": "Github token",
diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 03d8e55f0..a18d7462e 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -8,7 +8,11 @@ class github_workflows(github):
     watched_events = ["CODE_REPOSITORY"]
     produced_events = ["FILESYSTEM"]
     flags = ["passive", "safe"]
-    meta = {"description": "Download a github repositories workflow logs"}
+    meta = {
+        "description": "Download a github repositories workflow logs",
+        "created_date": "2024-04-29",
+        "author": "@domwhewell-sage"
+        }
     options = {"api_key": "", "num_logs": 1}
     options_desc = {
         "api_key": "Github token",
diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py
index 1369efa78..61a600bf4 100644
--- a/bbot/modules/gitlab.py
+++ b/bbot/modules/gitlab.py
@@ -5,7 +5,11 @@ class gitlab(BaseModule):
     watched_events = ["HTTP_RESPONSE", "TECHNOLOGY", "SOCIAL"]
     produced_events = ["TECHNOLOGY", "SOCIAL", "CODE_REPOSITORY", "FINDING"]
     flags = ["active", "safe"]
-    meta = {"description": "Detect GitLab instances and query them for repositories"}
+    meta = {
+        "description": "Detect GitLab instances and query them for repositories",
+        "created_date": "2024-03-11",
+        "author": "@TheTechromancer"
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "Gitlab access token"}
 
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 3271ef93f..4d9397f11 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -10,7 +10,11 @@ class gowitness(BaseModule):
     watched_events = ["URL", "SOCIAL"]
     produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"]
     flags = ["active", "safe", "web-screenshots"]
-    meta = {"description": "Take screenshots of webpages"}
+    meta = {
+        "description": "Take screenshots of webpages",
+        "created_date": "2022-07-08",
+        "author": "@TheTechromancer"
+        }
     options = {
         "version": "2.4.2",
         "threads": 4,
diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py
index d23f5c6cf..b4d86ceb4 100644
--- a/bbot/modules/hackertarget.py
+++ b/bbot/modules/hackertarget.py
@@ -5,7 +5,11 @@ class hackertarget(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the hackertarget.com API for subdomains"}
+    meta = {
+        "description": "Query the hackertarget.com API for subdomains",
+        "created_date": "2022-07-28",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://api.hackertarget.com"
 
diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py
index 4adaa766a..d51666f7c 100644
--- a/bbot/modules/host_header.py
+++ b/bbot/modules/host_header.py
@@ -6,7 +6,11 @@ class host_header(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {"description": "Try common HTTP Host header spoofing techniques"}
+    meta = {
+        "description": "Try common HTTP Host header spoofing techniques",
+        "created_date": "2022-07-27",
+        "author": "@liquidsec"
+        }
 
     in_scope_only = True
     per_hostport_only = True
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index 2e5dc0ffc..e11a49c7f 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -11,7 +11,11 @@ class httpx(BaseModule):
     watched_events = ["OPEN_TCP_PORT", "URL_UNVERIFIED", "URL"]
     produced_events = ["URL", "HTTP_RESPONSE"]
     flags = ["active", "safe", "web-basic", "web-thorough", "social-enum", "subdomain-enum", "cloud-enum"]
-    meta = {"description": "Visit webpages. Many other modules rely on httpx"}
+    meta = {
+        "description": "Visit webpages. Many other modules rely on httpx",
+        "created_date": "2022-07-08",
+        "author": "@TheTechromancer"
+        }
 
     options = {
         "threads": 50,
diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py
index add45b665..8ba6e9260 100644
--- a/bbot/modules/hunt.py
+++ b/bbot/modules/hunt.py
@@ -275,7 +275,11 @@ class hunt(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING"]
     flags = ["active", "safe", "web-thorough"]
-    meta = {"description": "Watch for commonly-exploitable HTTP parameters"}
+    meta = {
+        "description": "Watch for commonly-exploitable HTTP parameters",
+        "created_date": "2022-07-20",
+        "author": "@liquidsec"
+        }
     # accept all events regardless of scope distance
     scope_distance_modifier = None
 
diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py
index 792ca6d98..d3f3a27d9 100644
--- a/bbot/modules/hunterio.py
+++ b/bbot/modules/hunterio.py
@@ -5,7 +5,12 @@ class hunterio(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["EMAIL_ADDRESS", "DNS_NAME", "URL_UNVERIFIED"]
     flags = ["passive", "email-enum", "subdomain-enum", "safe"]
-    meta = {"description": "Query hunter.io for emails", "auth_required": True}
+    meta = {
+        "description": "Query hunter.io for emails",
+        "created_date": "2022-04-25",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "Hunter.IO API key"}
 
diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index 7d558a23a..56373638e 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -17,7 +17,11 @@ class iis_shortnames(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_HINT"]
     flags = ["active", "safe", "web-basic", "web-thorough", "iis-shortnames"]
-    meta = {"description": "Check for IIS shortname vulnerability"}
+    meta = {
+        "description": "Check for IIS shortname vulnerability",
+        "created_date": "2022-04-15",
+        "author": "@pmueller"
+        }
     options = {"detect_only": True, "max_node_count": 50}
     options_desc = {
         "detect_only": "Only detect the vulnerability and do not run the shortname scanner",
diff --git a/bbot/modules/internal/aggregate.py b/bbot/modules/internal/aggregate.py
index b1f11b04e..4a17390b6 100644
--- a/bbot/modules/internal/aggregate.py
+++ b/bbot/modules/internal/aggregate.py
@@ -3,7 +3,11 @@
 
 class aggregate(BaseReportModule):
     flags = ["passive", "safe"]
-    meta = {"description": "Summarize statistics at the end of a scan"}
+    meta = {
+        "description": "Summarize statistics at the end of a scan",
+        "created_date": "2022-07-25",
+        "author": "@TheTechromancer"
+        }
 
     async def report(self):
         self.log_table(*self.scan.stats._make_table(), table_name="scan-stats")
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 40067c69e..da2f3aa4f 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -338,7 +338,11 @@ class excavate(BaseInternalModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["passive"]
-    meta = {"description": "Passively extract juicy tidbits from scan data"}
+    meta = {
+        "description": "Passively extract juicy tidbits from scan data",
+        "created_date": "2022-06-27",
+        "author": "@liquidsec"
+        }
 
     scope_distance_modifier = None
 
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index d5b65af4d..6aee68a1d 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -26,7 +26,11 @@ class speculate(BaseInternalModule):
     ]
     produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"]
     flags = ["passive"]
-    meta = {"description": "Derive certain event types from others by common sense"}
+    meta = {
+        "description": "Derive certain event types from others by common sense",
+        "created_date": "2022-05-03",
+        "author": "@liquidsec"
+        }
 
     options = {"max_hosts": 65536, "ports": "80,443"}
     options_desc = {
diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py
index 847db0c7a..796299fc1 100644
--- a/bbot/modules/internetdb.py
+++ b/bbot/modules/internetdb.py
@@ -38,7 +38,11 @@ class internetdb(BaseModule):
     watched_events = ["IP_ADDRESS", "DNS_NAME"]
     produced_events = ["TECHNOLOGY", "VULNERABILITY", "FINDING", "OPEN_TCP_PORT", "DNS_NAME"]
     flags = ["passive", "safe", "portscan", "subdomain-enum"]
-    meta = {"description": "Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities"}
+    meta = {
+        "description": "Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities",
+        "created_date": "2023-12-22",
+        "author": "@TheTechromancer"
+        }
 
     _qsize = 500
 
diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py
index e192b2abb..be4195bab 100644
--- a/bbot/modules/ip2location.py
+++ b/bbot/modules/ip2location.py
@@ -9,7 +9,12 @@ class IP2Location(BaseModule):
     watched_events = ["IP_ADDRESS"]
     produced_events = ["GEOLOCATION"]
     flags = ["passive", "safe"]
-    meta = {"description": "Query IP2location.io's API for geolocation information. ", "auth_required": True}
+    meta = {
+        "description": "Query IP2location.io's API for geolocation information. ",
+        "created_date": "2023-09-12",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": "", "lang": ""}
     options_desc = {
         "api_key": "IP2location.io API Key",
diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py
index fe9673f5b..9586c707e 100644
--- a/bbot/modules/ipneighbor.py
+++ b/bbot/modules/ipneighbor.py
@@ -7,7 +7,11 @@ class ipneighbor(BaseModule):
     watched_events = ["IP_ADDRESS"]
     produced_events = ["IP_ADDRESS"]
     flags = ["passive", "subdomain-enum", "aggressive"]
-    meta = {"description": "Look beside IPs in their surrounding subnet"}
+    meta = {
+        "description": "Look beside IPs in their surrounding subnet",
+        "created_date": "2022-06-08",
+        "author": "@TheTechromancer"
+        }
     options = {"num_bits": 4}
     options_desc = {"num_bits": "Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)"}
     scope_distance_modifier = 1
diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py
index 98f139505..5a6f68759 100644
--- a/bbot/modules/ipstack.py
+++ b/bbot/modules/ipstack.py
@@ -10,7 +10,12 @@ class Ipstack(BaseModule):
     watched_events = ["IP_ADDRESS"]
     produced_events = ["GEOLOCATION"]
     flags = ["passive", "safe"]
-    meta = {"description": "Query IPStack's GeoIP API", "auth_required": True}
+    meta = {
+        "description": "Query IPStack's GeoIP API",
+        "created_date": "2022-11-26",
+        "author": "@tycoonslive",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "IPStack GeoIP API Key"}
     scope_distance_modifier = 1
diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py
index 45053755a..f2e1b1011 100644
--- a/bbot/modules/leakix.py
+++ b/bbot/modules/leakix.py
@@ -8,7 +8,11 @@ class leakix(subdomain_enum_apikey):
     options = {"api_key": ""}
     # NOTE: API key is not required (but having one will get you more results)
     options_desc = {"api_key": "LeakIX API Key"}
-    meta = {"description": "Query leakix.net for subdomains"}
+    meta = {
+        "description": "Query leakix.net for subdomains",
+        "created_date": "2022-07-11",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://leakix.net"
 
diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py
index ddeb76163..10e0cd921 100644
--- a/bbot/modules/masscan.py
+++ b/bbot/modules/masscan.py
@@ -8,7 +8,11 @@ class masscan(portscanner):
     flags = ["active", "portscan", "aggressive"]
     watched_events = ["IP_ADDRESS", "IP_RANGE"]
     produced_events = ["OPEN_TCP_PORT"]
-    meta = {"description": "Port scan with masscan. By default, scans top 100 ports."}
+    meta = {
+        "description": "Port scan with masscan. By default, scans top 100 ports.",
+        "created_date": "2023-01-27",
+        "author": "@TheTechromancer"
+        }
     options = {
         "top_ports": 100,
         "ports": "",
diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py
index e453ee8d4..7e3f57a7c 100644
--- a/bbot/modules/massdns.py
+++ b/bbot/modules/massdns.py
@@ -25,7 +25,11 @@ class massdns(subdomain_enum):
     flags = ["subdomain-enum", "passive", "aggressive"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Brute-force subdomains with massdns (highly effective)"}
+    meta = {
+        "description": "Brute-force subdomains with massdns (highly effective)",
+        "created_date": "2023-03-29",
+        "author": "@TheTechromancer"
+        }
     options = {
         "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt",
         "max_resolvers": 1000,
diff --git a/bbot/modules/myssl.py b/bbot/modules/myssl.py
index a08c885ed..5527cfa88 100644
--- a/bbot/modules/myssl.py
+++ b/bbot/modules/myssl.py
@@ -5,7 +5,11 @@ class myssl(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query myssl.com's API for subdomains"}
+    meta = {
+        "description": "Query myssl.com's API for subdomains",
+        "created_date": "2023-07-10",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://myssl.com/api/v1/discover_sub_domain"
 
diff --git a/bbot/modules/newsletters.py b/bbot/modules/newsletters.py
index 726d633fe..f8dd33a65 100644
--- a/bbot/modules/newsletters.py
+++ b/bbot/modules/newsletters.py
@@ -19,7 +19,11 @@ class newsletters(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING"]
     flags = ["active", "safe"]
-    meta = {"description": "Searches for Newsletter Submission Entry Fields on Websites"}
+    meta = {
+        "description": "Searches for Newsletter Submission Entry Fields on Websites",
+        "created_date": "2024-02-02",
+        "author": "@stryker2k2"
+        }
 
     # Parse through Website to find a Text Entry Box of 'type = email'
     # and ensure that there is placeholder text within it.
diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py
index ccdb0974e..ec1723ac6 100644
--- a/bbot/modules/nmap.py
+++ b/bbot/modules/nmap.py
@@ -6,7 +6,11 @@ class nmap(portscanner):
     watched_events = ["IP_ADDRESS", "DNS_NAME", "IP_RANGE"]
     produced_events = ["OPEN_TCP_PORT"]
     flags = ["active", "portscan", "aggressive", "web-thorough"]
-    meta = {"description": "Port scan with nmap. By default, scans top 100 ports."}
+    meta = {
+        "description": "Port scan with nmap. By default, scans top 100 ports.",
+        "created_date": "2022-03-12",
+        "author": "@TheTechromancer"
+        }
     options = {
         "top_ports": 100,
         "ports": "",
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index c69beb941..718b800e2 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -69,7 +69,11 @@ class ntlm(BaseModule):
     watched_events = ["URL", "HTTP_RESPONSE"]
     produced_events = ["FINDING", "DNS_NAME"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {"description": "Watch for HTTP endpoints that support NTLM authentication"}
+    meta = {
+        "description": "Watch for HTTP endpoints that support NTLM authentication",
+        "created_date": "2022-07-25",
+        "author": "@liquidsec"
+        }
     options = {"try_all": False}
     options_desc = {"try_all": "Try every NTLM endpoint"}
 
diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py
index 13a483aad..5096e3cd4 100644
--- a/bbot/modules/oauth.py
+++ b/bbot/modules/oauth.py
@@ -7,7 +7,11 @@ class OAUTH(BaseModule):
     watched_events = ["DNS_NAME", "URL_UNVERIFIED"]
     produced_events = ["DNS_NAME"]
     flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "web-thorough", "active", "safe"]
-    meta = {"description": "Enumerate OAUTH and OpenID Connect services"}
+    meta = {
+        "description": "Enumerate OAUTH and OpenID Connect services",
+        "created_date": "2023-07-12",
+        "author": "@TheTechromancer"
+        }
     options = {"try_all": False}
     options_desc = {"try_all": "Check for OAUTH/IODC on every subdomain and URL."}
 
diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py
index 72f2e1d5b..bf60420e4 100644
--- a/bbot/modules/otx.py
+++ b/bbot/modules/otx.py
@@ -5,7 +5,11 @@ class otx(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query otx.alienvault.com for subdomains"}
+    meta = {
+        "description": "Query otx.alienvault.com for subdomains",
+        "created_date": "2022-08-24",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://otx.alienvault.com"
 
diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py
index 0372dc1bd..4d624588a 100644
--- a/bbot/modules/output/asset_inventory.py
+++ b/bbot/modules/output/asset_inventory.py
@@ -33,7 +33,11 @@ class asset_inventory(CSV):
         "HTTP_RESPONSE",
     ]
     produced_events = ["IP_ADDRESS", "OPEN_TCP_PORT"]
-    meta = {"description": "Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV"}
+    meta = {
+        "description": "Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV",
+        "created_date": "2022-09-30",
+        "author": "@liquidsec"
+        }
     options = {"output_file": "", "use_previous": False, "recheck": False, "summary_netmask": 16}
     options_desc = {
         "output_file": "Set a custom output file",
diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py
index fe8af7e89..ae403b34f 100644
--- a/bbot/modules/output/csv.py
+++ b/bbot/modules/output/csv.py
@@ -6,7 +6,11 @@
 
 class CSV(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output to CSV"}
+    meta = {
+        "description": "Output to CSV",
+        "created_date": "2022-04-07",
+        "author": "@TheTechromancer"
+        }
     options = {"output_file": ""}
     options_desc = {"output_file": "Output to CSV file"}
 
diff --git a/bbot/modules/output/discord.py b/bbot/modules/output/discord.py
index 06684bd30..8df91eb7d 100644
--- a/bbot/modules/output/discord.py
+++ b/bbot/modules/output/discord.py
@@ -3,7 +3,11 @@
 
 class Discord(WebhookOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Message a Discord channel when certain events are encountered"}
+    meta = {
+        "description": "Message a Discord channel when certain events are encountered",
+        "created_date": "2023-08-14",
+        "author": "@TheTechromancer"
+        }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
         "webhook_url": "Discord webhook URL",
diff --git a/bbot/modules/output/emails.py b/bbot/modules/output/emails.py
index e96c5d97c..0e13926de 100644
--- a/bbot/modules/output/emails.py
+++ b/bbot/modules/output/emails.py
@@ -5,7 +5,11 @@
 class Emails(Human):
     watched_events = ["EMAIL_ADDRESS"]
     flags = ["email-enum"]
-    meta = {"description": "Output any email addresses found belonging to the target domain"}
+    meta = {
+        "description": "Output any email addresses found belonging to the target domain",
+        "created_date": "2023-12-23",
+        "author": "@domwhewell-sage"
+        }
     options = {"output_file": ""}
     options_desc = {"output_file": "Output to file"}
     in_scope_only = True
diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py
index 014610736..890f55343 100644
--- a/bbot/modules/output/http.py
+++ b/bbot/modules/output/http.py
@@ -5,7 +5,11 @@
 
 class HTTP(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Send every event to a custom URL via a web request"}
+    meta = {
+        "description": "Send every event to a custom URL via a web request",
+        "created_date": "2022-04-13",
+        "author": "@TheTechromancer"
+        }
     options = {
         "url": "",
         "method": "POST",
diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py
index e1f4746c4..11c593bee 100644
--- a/bbot/modules/output/human.py
+++ b/bbot/modules/output/human.py
@@ -6,7 +6,11 @@
 
 class Human(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output to text"}
+    meta = {
+        "description": "Output to text",
+        "created_date": "2022-04-07",
+        "author": "@TheTechromancer"
+        }
     options = {"output_file": "", "console": True}
     options_desc = {"output_file": "Output to file", "console": "Output to console"}
     vuln_severity_map = {"LOW": "HUGEWARNING", "MEDIUM": "HUGEWARNING", "HIGH": "CRITICAL", "CRITICAL": "CRITICAL"}
diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py
index bf8517db9..34f45bbc1 100644
--- a/bbot/modules/output/json.py
+++ b/bbot/modules/output/json.py
@@ -6,7 +6,11 @@
 
 class JSON(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output to Newline-Delimited JSON (NDJSON)"}
+    meta = {
+        "description": "Output to Newline-Delimited JSON (NDJSON)",
+        "created_date": "2022-04-07",
+        "author": "@TheTechromancer"
+        }
     options = {"output_file": "", "console": False, "siem_friendly": False}
     options_desc = {
         "output_file": "Output to file",
diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py
index 2cc083544..711c55d3e 100644
--- a/bbot/modules/output/neo4j.py
+++ b/bbot/modules/output/neo4j.py
@@ -25,7 +25,11 @@ class neo4j(BaseOutputModule):
     """
 
     watched_events = ["*"]
-    meta = {"description": "Output to Neo4j"}
+    meta = {
+        "description": "Output to Neo4j",
+        "created_date": "2022-04-07",
+        "author": "@TheTechromancer"
+        }
     options = {"uri": "bolt://localhost:7687", "username": "neo4j", "password": "bbotislife"}
     options_desc = {
         "uri": "Neo4j server + port",
diff --git a/bbot/modules/output/python.py b/bbot/modules/output/python.py
index 3aebfeb52..27740a9bb 100644
--- a/bbot/modules/output/python.py
+++ b/bbot/modules/output/python.py
@@ -3,7 +3,11 @@
 
 class python(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output via Python API"}
+    meta = {
+        "description": "Output via Python API",
+        "created_date": "2022-09-13",
+        "author": "@TheTechromancer"
+        }
 
     async def _worker(self):
         pass
diff --git a/bbot/modules/output/slack.py b/bbot/modules/output/slack.py
index 1dda46942..fea4326a1 100644
--- a/bbot/modules/output/slack.py
+++ b/bbot/modules/output/slack.py
@@ -5,7 +5,11 @@
 
 class Slack(WebhookOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Message a Slack channel when certain events are encountered"}
+    meta = {
+        "description": "Message a Slack channel when certain events are encountered",
+        "created_date": "2023-08-14",
+        "author": "@TheTechromancer"
+        }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
         "webhook_url": "Discord webhook URL",
diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py
index 242f1759e..3a82d1d99 100644
--- a/bbot/modules/output/splunk.py
+++ b/bbot/modules/output/splunk.py
@@ -5,7 +5,11 @@
 
 class Splunk(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Send every event to a splunk instance through HTTP Event Collector"}
+    meta = {
+        "description": "Send every event to a splunk instance through HTTP Event Collector",
+        "created_date": "2024-02-17",
+        "author": "@w0Tx"
+        }
     options = {
         "url": "",
         "hectoken": "",
diff --git a/bbot/modules/output/subdomains.py b/bbot/modules/output/subdomains.py
index bfb7174ac..779f806dc 100644
--- a/bbot/modules/output/subdomains.py
+++ b/bbot/modules/output/subdomains.py
@@ -5,7 +5,11 @@
 class Subdomains(Human):
     watched_events = ["DNS_NAME", "DNS_NAME_UNRESOLVED"]
     flags = ["subdomain-enum"]
-    meta = {"description": "Output only resolved, in-scope subdomains"}
+    meta = {
+        "description": "Output only resolved, in-scope subdomains",
+        "created_date": "2023-07-31",
+        "author": "@TheTechromancer"
+        }
     options = {"output_file": "", "include_unresolved": False}
     options_desc = {"output_file": "Output to file", "include_unresolved": "Include unresolved subdomains in output"}
     accept_dupes = False
diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py
index 3f018038d..8659a9eed 100644
--- a/bbot/modules/output/teams.py
+++ b/bbot/modules/output/teams.py
@@ -3,7 +3,11 @@
 
 class Teams(WebhookOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Message a Teams channel when certain events are encountered"}
+    meta = {
+        "description": "Message a Teams channel when certain events are encountered",
+        "created_date": "2023-08-14",
+        "author": "@TheTechromancer"
+        }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
         "webhook_url": "Discord webhook URL",
diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py
index 793f26c32..e4086a449 100644
--- a/bbot/modules/output/web_report.py
+++ b/bbot/modules/output/web_report.py
@@ -5,7 +5,11 @@
 
 class web_report(BaseOutputModule):
     watched_events = ["URL", "TECHNOLOGY", "FINDING", "VULNERABILITY", "VHOST"]
-    meta = {"description": "Create a markdown report with web assets"}
+    meta = {
+        "description": "Create a markdown report with web assets",
+        "created_date": "2023-02-08",
+        "author": "@liquidsec"
+        }
     options = {
         "output_file": "",
         "css_theme_file": "https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css",
diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py
index 76edcba4b..f3ef57a85 100644
--- a/bbot/modules/output/websocket.py
+++ b/bbot/modules/output/websocket.py
@@ -7,7 +7,11 @@
 
 class Websocket(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output to websockets"}
+    meta = {
+        "description": "Output to websockets",
+        "created_date": "2022-04-15",
+        "author": "@TheTechromancer"
+        }
     options = {"url": "", "token": "", "preserve_graph": True}
     options_desc = {
         "url": "Web URL",
diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py
index 251931f58..ea671929a 100644
--- a/bbot/modules/paramminer_cookies.py
+++ b/bbot/modules/paramminer_cookies.py
@@ -11,6 +11,8 @@ class paramminer_cookies(paramminer_headers):
     flags = ["active", "aggressive", "slow", "web-paramminer"]
     meta = {
         "description": "Smart brute-force to check for common HTTP cookie parameters",
+        "created_date": "2022-06-27",
+        "author": "@liquidsec"
     }
     options = {
         "wordlist": "",  # default is defined within setup function
diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py
index 7891e05cf..e4fdc2949 100644
--- a/bbot/modules/paramminer_getparams.py
+++ b/bbot/modules/paramminer_getparams.py
@@ -9,7 +9,11 @@ class paramminer_getparams(paramminer_headers):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "slow", "web-paramminer"]
-    meta = {"description": "Use smart brute-force to check for common HTTP GET parameters"}
+    meta = {
+        "description": "Use smart brute-force to check for common HTTP GET parameters",
+        "created_date": "2022-06-28",
+        "author": "@liquidsec"
+        }
     scanned_hosts = []
     options = {
         "wordlist": "",  # default is defined within setup function
diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index c7418912f..6979e1330 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -11,7 +11,11 @@ class paramminer_headers(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "slow", "web-paramminer"]
-    meta = {"description": "Use smart brute-force to check for common HTTP header parameters"}
+    meta = {
+        "description": "Use smart brute-force to check for common HTTP header parameters",
+        "created_date": "2022-04-15",
+        "author": "@pmueller"
+        }
     options = {
         "wordlist": "",  # default is defined within setup function
         "http_extract": True,
diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py
index dfed92939..999085baa 100644
--- a/bbot/modules/passivetotal.py
+++ b/bbot/modules/passivetotal.py
@@ -5,7 +5,12 @@ class passivetotal(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the PassiveTotal API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query the PassiveTotal API for subdomains",
+        "created_date": "2022-08-08",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"username": "", "api_key": ""}
     options_desc = {"username": "RiskIQ Username", "api_key": "RiskIQ API Key"}
 
diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py
index 2c378f585..5b52bbd14 100644
--- a/bbot/modules/pgp.py
+++ b/bbot/modules/pgp.py
@@ -5,7 +5,11 @@ class pgp(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["EMAIL_ADDRESS"]
     flags = ["passive", "email-enum", "safe"]
-    meta = {"description": "Query common PGP servers for email addresses"}
+    meta = {
+        "description": "Query common PGP servers for email addresses",
+        "created_date": "2022-08-10",
+        "author": "@TheTechromancer"
+        }
     options = {
         "search_urls": [
             "https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=<query>",
diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py
index 5a63f824e..feff85c1f 100644
--- a/bbot/modules/postman.py
+++ b/bbot/modules/postman.py
@@ -5,7 +5,11 @@ class postman(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["passive", "subdomain-enum", "safe"]
-    meta = {"description": "Query Postman's API for related workspaces, collections, requests"}
+    meta = {
+        "description": "Query Postman's API for related workspaces, collections, requests",
+        "created_date": "2023-12-23",
+        "author": "@domwhewell-sage"
+        }
 
     base_url = "https://www.postman.com/_api"
 
diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py
index 088288ddb..05b807773 100644
--- a/bbot/modules/rapiddns.py
+++ b/bbot/modules/rapiddns.py
@@ -5,7 +5,11 @@ class rapiddns(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query rapiddns.io for subdomains"}
+    meta = {
+        "description": "Query rapiddns.io for subdomains",
+        "created_date": "2022-08-24",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://rapiddns.io"
 
diff --git a/bbot/modules/report/affiliates.py b/bbot/modules/report/affiliates.py
index bb6323664..56357bee0 100644
--- a/bbot/modules/report/affiliates.py
+++ b/bbot/modules/report/affiliates.py
@@ -5,7 +5,11 @@ class affiliates(BaseReportModule):
     watched_events = ["*"]
     produced_events = []
     flags = ["passive", "safe", "affiliates"]
-    meta = {"description": "Summarize affiliate domains at the end of a scan"}
+    meta = {
+        "description": "Summarize affiliate domains at the end of a scan",
+        "created_date": "2022-07-25",
+        "author": "@TheTechromancer"
+        }
     scope_distance_modifier = None
     accept_dupes = True
 
diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py
index f906c785e..4802c5659 100644
--- a/bbot/modules/report/asn.py
+++ b/bbot/modules/report/asn.py
@@ -5,7 +5,11 @@ class asn(BaseReportModule):
     watched_events = ["IP_ADDRESS"]
     produced_events = ["ASN"]
     flags = ["passive", "subdomain-enum", "safe"]
-    meta = {"description": "Query ripe and bgpview.io for ASNs"}
+    meta = {
+        "description": "Query ripe and bgpview.io for ASNs",
+        "created_date": "2022-07-25",
+        "author": "@TheTechromancer"
+        }
     scope_distance_modifier = 1
     # we accept dupes to avoid missing data
     # because sometimes IP addresses are re-emitted with lower scope distances
diff --git a/bbot/modules/riddler.py b/bbot/modules/riddler.py
index d525acbad..a5c2badac 100644
--- a/bbot/modules/riddler.py
+++ b/bbot/modules/riddler.py
@@ -5,7 +5,11 @@ class riddler(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query riddler.io for subdomains"}
+    meta = {
+        "description": "Query riddler.io for subdomains",
+        "created_date": "2022-08-25",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://riddler.io"
 
diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py
index 717900bee..57d65503c 100644
--- a/bbot/modules/robots.py
+++ b/bbot/modules/robots.py
@@ -5,7 +5,11 @@ class robots(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {"description": "Look for and parse robots.txt"}
+    meta = {
+        "description": "Look for and parse robots.txt",
+        "created_date": "2023-02-01",
+        "author": "@liquidsec"
+        }
 
     options = {"include_sitemap": False, "include_allow": True, "include_disallow": True}
     options_desc = {
diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py
index 3fc8ad539..9facf5ff3 100644
--- a/bbot/modules/secretsdb.py
+++ b/bbot/modules/secretsdb.py
@@ -8,7 +8,11 @@ class secretsdb(BaseModule):
     watched_events = ["HTTP_RESPONSE"]
     produced_events = ["FINDING"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {"description": "Detect common secrets with secrets-patterns-db"}
+    meta = {
+        "description": "Detect common secrets with secrets-patterns-db",
+        "created_date": "2023-03-17",
+        "author": "@TheTechromancer"
+        }
     options = {
         "min_confidence": 99,
         "signatures": "https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml",
diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py
index 8d1d1b6a0..7899b55ee 100644
--- a/bbot/modules/securitytrails.py
+++ b/bbot/modules/securitytrails.py
@@ -5,7 +5,12 @@ class securitytrails(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query the SecurityTrails API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query the SecurityTrails API for subdomains",
+        "created_date": "2022-07-03",
+        "author": "@TheTechromancer"
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "SecurityTrails API key"}
 
diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py
index 4d19b237e..a4248cbdc 100644
--- a/bbot/modules/shodan_dns.py
+++ b/bbot/modules/shodan_dns.py
@@ -5,7 +5,12 @@ class shodan_dns(shodan):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query Shodan for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query Shodan for subdomains",
+        "created_date": "2022-07-03",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "Shodan API key"}
 
diff --git a/bbot/modules/sitedossier.py b/bbot/modules/sitedossier.py
index 86872c052..897201ac3 100644
--- a/bbot/modules/sitedossier.py
+++ b/bbot/modules/sitedossier.py
@@ -5,7 +5,11 @@ class sitedossier(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query sitedossier.com for subdomains"}
+    meta = {
+        "description": "Query sitedossier.com for subdomains",
+        "created_date": "2023-08-04",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "http://www.sitedossier.com/parentdomain"
     max_pages = 10
diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py
index 4bd76c70d..371cd39d3 100644
--- a/bbot/modules/skymem.py
+++ b/bbot/modules/skymem.py
@@ -7,7 +7,11 @@ class skymem(emailformat):
     watched_events = ["DNS_NAME"]
     produced_events = ["EMAIL_ADDRESS"]
     flags = ["passive", "email-enum", "safe"]
-    meta = {"description": "Query skymem.info for email addresses"}
+    meta = {
+        "description": "Query skymem.info for email addresses",
+        "created_date": "2022-07-11",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://www.skymem.info"
 
diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py
index 8ee344452..b921894f2 100644
--- a/bbot/modules/smuggler.py
+++ b/bbot/modules/smuggler.py
@@ -12,7 +12,11 @@ class smuggler(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "slow", "web-thorough"]
-    meta = {"description": "Check for HTTP smuggling"}
+    meta = {
+        "description": "Check for HTTP smuggling",
+        "created_date": "2022-07-06",
+        "author": "@liquidsec"
+        }
 
     in_scope_only = True
     per_hostport_only = True
diff --git a/bbot/modules/social.py b/bbot/modules/social.py
index 3688d7402..fdb106e8b 100644
--- a/bbot/modules/social.py
+++ b/bbot/modules/social.py
@@ -5,7 +5,11 @@
 class social(BaseModule):
     watched_events = ["URL_UNVERIFIED"]
     produced_events = ["SOCIAL"]
-    meta = {"description": "Look for social media links in webpages"}
+    meta = {
+        "description": "Look for social media links in webpages",
+        "created_date": "2023-03-28",
+        "author": "@TheTechromancer"
+        }
     flags = ["passive", "safe", "social-enum"]
 
     # platform name : (regex, case_sensitive)
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index caca5feca..03cf3b300 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -13,6 +13,8 @@ class sslcert(BaseModule):
     flags = ["affiliates", "subdomain-enum", "email-enum", "active", "safe", "web-basic", "web-thorough"]
     meta = {
         "description": "Visit open ports and retrieve SSL certificates",
+        "created_date": "2022-03-30",
+        "author": "@TheTechromancer"
     }
     options = {"timeout": 5.0, "skip_non_ssl": True}
     options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"}
diff --git a/bbot/modules/subdomaincenter.py b/bbot/modules/subdomaincenter.py
index c5c69293c..85f0ecbd7 100644
--- a/bbot/modules/subdomaincenter.py
+++ b/bbot/modules/subdomaincenter.py
@@ -5,7 +5,11 @@ class subdomaincenter(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
-    meta = {"description": "Query subdomain.center's API for subdomains"}
+    meta = {
+        "description": "Query subdomain.center's API for subdomains",
+        "created_date": "2023-07-26",
+        "author": "@TheTechromancer"
+        }
 
     base_url = "https://api.subdomain.center"
     retries = 2
diff --git a/bbot/modules/sublist3r.py b/bbot/modules/sublist3r.py
index 3c13cf308..0a673bad7 100644
--- a/bbot/modules/sublist3r.py
+++ b/bbot/modules/sublist3r.py
@@ -8,6 +8,8 @@ class sublist3r(subdomain_enum):
     flags = ["passive", "safe"]
     meta = {
         "description": "Query sublist3r's API for subdomains",
+        "created_date": "2022-03-29",
+        "author": "@Want-EyeTea"
     }
 
     base_url = "https://api.sublist3r.com/search.php"
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index c86da8897..b70a3b054 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -9,7 +9,11 @@ class telerik(BaseModule):
     watched_events = ["URL", "HTTP_RESPONSE"]
     produced_events = ["VULNERABILITY", "FINDING"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {"description": "Scan for critical Telerik vulnerabilities"}
+    meta = {
+        "description": "Scan for critical Telerik vulnerabilities",
+        "created_date": "2022-04-10",
+        "author": "@liquidsec"
+        }
 
     telerikVersions = [
         "2007.1423",
diff --git a/bbot/modules/threatminer.py b/bbot/modules/threatminer.py
index bbc1e23c3..c3e5cd958 100644
--- a/bbot/modules/threatminer.py
+++ b/bbot/modules/threatminer.py
@@ -7,6 +7,8 @@ class threatminer(subdomain_enum):
     flags = ["subdomain-enum", "passive", "safe"]
     meta = {
         "description": "Query threatminer's API for subdomains",
+        "created_date": "2022-07-28",
+        "author": "@TheTechromancer"
     }
 
     base_url = "https://api.threatminer.org/v2"
diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index 5acb45a12..8fb05c47f 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -6,7 +6,11 @@ class trufflehog(BaseModule):
     watched_events = ["FILESYSTEM"]
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["passive", "safe"]
-    meta = {"description": "TruffleHog is a tool for finding credentials"}
+    meta = {
+        "description": "TruffleHog is a tool for finding credentials",
+        "created_date": "2024-03-12",
+        "author": "@domwhewell-sage"
+        }
 
     options = {
         "version": "3.75.1",
diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py
index 983595fe1..9f17c8157 100644
--- a/bbot/modules/url_manipulation.py
+++ b/bbot/modules/url_manipulation.py
@@ -6,7 +6,11 @@ class url_manipulation(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {"description": "Attempt to identify URL parsing/routing based vulnerabilities"}
+    meta = {
+        "description": "Attempt to identify URL parsing/routing based vulnerabilities",
+        "created_date": "2022-09-27",
+        "author": "@liquidsec"
+        }
     in_scope_only = True
 
     options = {"allow_redirects": True}
diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py
index 4c3811af0..49b6ca0ee 100644
--- a/bbot/modules/urlscan.py
+++ b/bbot/modules/urlscan.py
@@ -7,6 +7,8 @@ class urlscan(subdomain_enum):
     produced_events = ["DNS_NAME", "URL_UNVERIFIED"]
     meta = {
         "description": "Query urlscan.io for subdomains",
+        "created_date": "2022-06-09",
+        "author": "@TheTechromancer"
     }
     options = {"urls": False}
     options_desc = {"urls": "Emit URLs in addition to DNS_NAMEs"}
diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py
index 0d996bf09..6f382bf2a 100644
--- a/bbot/modules/viewdns.py
+++ b/bbot/modules/viewdns.py
@@ -13,6 +13,8 @@ class viewdns(BaseModule):
     flags = ["affiliates", "passive", "safe"]
     meta = {
         "description": "Query viewdns.info's reverse whois for related domains",
+        "created_date": "2022-07-04",
+        "author": "@TheTechromancer"
     }
     base_url = "https://viewdns.info"
     in_scope_only = True
diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py
index 8e0c03934..323c4cfb8 100644
--- a/bbot/modules/virustotal.py
+++ b/bbot/modules/virustotal.py
@@ -5,7 +5,12 @@ class virustotal(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query VirusTotal's API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query VirusTotal's API for subdomains",
+        "created_date": "2022-08-25",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": ""}
     options_desc = {"api_key": "VirusTotal API Key"}
 
diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py
index a91bfd920..5af886aeb 100644
--- a/bbot/modules/wafw00f.py
+++ b/bbot/modules/wafw00f.py
@@ -16,7 +16,11 @@ class wafw00f(BaseModule):
     watched_events = ["URL"]
     produced_events = ["WAF"]
     flags = ["active", "aggressive"]
-    meta = {"description": "Web Application Firewall Fingerprinting Tool"}
+    meta = {
+        "description": "Web Application Firewall Fingerprinting Tool",
+        "created_date": "2023-02-15",
+        "author": "@liquidsec"
+        }
 
     deps_pip = ["wafw00f~=2.2.0"]
 
diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py
index c87274a29..61a858b9f 100644
--- a/bbot/modules/wappalyzer.py
+++ b/bbot/modules/wappalyzer.py
@@ -16,6 +16,8 @@ class wappalyzer(BaseModule):
     flags = ["active", "safe", "web-basic", "web-thorough"]
     meta = {
         "description": "Extract technologies from web responses",
+        "created_date": "2022-04-15",
+        "author": "@liquidsec"
     }
     deps_pip = ["python-Wappalyzer~=0.3.1", "aiohttp~=3.9.0b0"]
     # accept all events regardless of scope distance
diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py
index 92dc78db5..75f45d405 100644
--- a/bbot/modules/wayback.py
+++ b/bbot/modules/wayback.py
@@ -9,6 +9,8 @@ class wayback(subdomain_enum):
     produced_events = ["URL_UNVERIFIED", "DNS_NAME"]
     meta = {
         "description": "Query archive.org's API for subdomains",
+        "created_date": "2022-04-01",
+        "author": "@pmueller"
     }
     options = {"urls": False, "garbage_threshold": 10}
     options_desc = {
diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py
index ec921bc25..75823d308 100644
--- a/bbot/modules/zoomeye.py
+++ b/bbot/modules/zoomeye.py
@@ -5,7 +5,12 @@ class zoomeye(subdomain_enum_apikey):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["affiliates", "subdomain-enum", "passive", "safe"]
-    meta = {"description": "Query ZoomEye's API for subdomains", "auth_required": True}
+    meta = {
+        "description": "Query ZoomEye's API for subdomains",
+        "created_date": "2022-08-03",
+        "author": "@TheTechromancer",
+        "auth_required": True
+        }
     options = {"api_key": "", "max_pages": 20, "include_related": False}
     options_desc = {
         "api_key": "ZoomEye API key",

From a703404ced834c00022c2b06ae8bebf2acaf0151 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 20 May 2024 16:30:55 -0400
Subject: [PATCH 170/220] debugging generic_ssrf test failures

---
 bbot/test/conftest.py                             | 15 +++++++--------
 bbot/test/test_step_2/module_tests/base.py        |  5 +++++
 .../module_tests/test_module_dotnetnuke.py        |  2 +-
 .../module_tests/test_module_generic_ssrf.py      | 11 +++++++++--
 .../module_tests/test_module_host_header.py       |  2 +-
 5 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 9d92a9315..50b5c765e 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -109,20 +109,19 @@ def bbot_httpserver_allinterfaces():
     server.clear()
 
 
-@pytest.fixture
-def interactsh_mock_instance():
-    interactsh_mock = Interactsh_mock()
-    return interactsh_mock
-
-
 class Interactsh_mock:
-    def __init__(self):
+    def __init__(self, name):
+        self.name = name
+        self.log = logging.getLogger(f"bbot.interactsh.{self.name}")
         self.interactions = []
         self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef"
         self.stop = False
         self.poll_task = None
 
-    def mock_interaction(self, subdomain_tag):
+    def mock_interaction(self, subdomain_tag, msg=None):
+        self.log.critical(f"Mocking interaction to subdomain tag: {subdomain_tag}")
+        if msg is not None:
+            self.log.critical(msg)
         self.interactions.append(subdomain_tag)
 
     async def register(self, callback=None):
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 1300c5ad3..8dcabe996 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -101,6 +101,11 @@ async def mock_dns(self, mock_data, scan=None):
                 scan = self.scan
             await scan.helpers.dns._mock_dns(mock_data)
 
+        def mock_interactsh(self, name):
+            from ...conftest import Interactsh_mock
+
+            return Interactsh_mock(name)
+
         @property
         def module(self):
             return self.scan.modules[self.name]
diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py
index f14900515..de78ad50b 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py
@@ -150,7 +150,7 @@ def request_handler(self, request):
 
     async def setup_before_prep(self, module_test):
 
-        self.interactsh_mock_instance = module_test.request_fixture.getfixturevalue("interactsh_mock_instance")
+        self.interactsh_mock_instance = module_test.mock_interactsh("dotnetnuke_blindssrf")
         module_test.monkeypatch.setattr(
             module_test.scan.helpers, "interactsh", lambda *args, **kwargs: self.interactsh_mock_instance
         )
diff --git a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
index 370dd151a..31d05fdc1 100644
--- a/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
+++ b/bbot/test/test_step_2/module_tests/test_module_generic_ssrf.py
@@ -16,19 +16,26 @@ class TestGeneric_SSRF(ModuleTestBase):
     modules_overrides = ["httpx", "generic_ssrf"]
 
     def request_handler(self, request):
+        self.log.critical(str(request))
+        self.log.critical(request.full_path)
+        self.log.critical(request.data.decode())
         subdomain_tag = None
 
         if request.method == "GET":
             subdomain_tag = extract_subdomain_tag(request.full_path)
         elif request.method == "POST":
             subdomain_tag = extract_subdomain_tag(request.data.decode())
+        self.log.critical(f"subdomain tag: {subdomain_tag}")
         if subdomain_tag:
-            self.interactsh_mock_instance.mock_interaction(subdomain_tag)
+            self.interactsh_mock_instance.mock_interaction(
+                subdomain_tag, msg=f"{request.method}: {request.data.decode()}"
+            )
 
         return Response("alive", status=200)
 
     async def setup_before_prep(self, module_test):
-        self.interactsh_mock_instance = module_test.request_fixture.getfixturevalue("interactsh_mock_instance")
+        self.interactsh_mock_instance = module_test.mock_interactsh("generic_ssrf")
+        self.interactsh_mock_instance.mock_interaction("asdf")
         module_test.monkeypatch.setattr(
             module_test.scan.helpers, "interactsh", lambda *args, **kwargs: self.interactsh_mock_instance
         )
diff --git a/bbot/test/test_step_2/module_tests/test_module_host_header.py b/bbot/test/test_step_2/module_tests/test_module_host_header.py
index 9741d9bc1..b71a31b1d 100644
--- a/bbot/test/test_step_2/module_tests/test_module_host_header.py
+++ b/bbot/test/test_step_2/module_tests/test_module_host_header.py
@@ -34,7 +34,7 @@ def request_handler(self, request):
         return Response(f"Alive, host is: defaulthost.com", status=200)
 
     async def setup_before_prep(self, module_test):
-        self.interactsh_mock_instance = module_test.request_fixture.getfixturevalue("interactsh_mock_instance")
+        self.interactsh_mock_instance = module_test.mock_interactsh("host_header")
         module_test.monkeypatch.setattr(
             module_test.scan.helpers, "interactsh", lambda *args, **kwargs: self.interactsh_mock_instance
         )

From 9dabae872380b4f36857dd744f7e1a8ef0b9e4a5 Mon Sep 17 00:00:00 2001
From: Amir <a@a.com>
Date: Mon, 20 May 2024 23:11:10 +0200
Subject: [PATCH 171/220] black

---
 bbot/modules/ajaxpro.py                | 4 ++--
 bbot/modules/anubisdb.py               | 4 ++--
 bbot/modules/azure_realm.py            | 4 ++--
 bbot/modules/azure_tenant.py           | 4 ++--
 bbot/modules/baddns.py                 | 4 ++--
 bbot/modules/baddns_zone.py            | 4 ++--
 bbot/modules/badsecrets.py             | 4 ++--
 bbot/modules/bevigil.py                | 4 ++--
 bbot/modules/binaryedge.py             | 4 ++--
 bbot/modules/bucket_amazon.py          | 4 ++--
 bbot/modules/bucket_azure.py           | 4 ++--
 bbot/modules/bucket_digitalocean.py    | 4 ++--
 bbot/modules/bucket_file_enum.py       | 2 +-
 bbot/modules/bucket_firebase.py        | 4 ++--
 bbot/modules/bucket_google.py          | 4 ++--
 bbot/modules/builtwith.py              | 4 ++--
 bbot/modules/bypass403.py              | 6 +-----
 bbot/modules/c99.py                    | 4 ++--
 bbot/modules/censys.py                 | 4 ++--
 bbot/modules/certspotter.py            | 4 ++--
 bbot/modules/chaos.py                  | 4 ++--
 bbot/modules/code_repository.py        | 4 ++--
 bbot/modules/columbus.py               | 4 ++--
 bbot/modules/crobat.py                 | 6 +-----
 bbot/modules/crt.py                    | 4 ++--
 bbot/modules/deadly/dastardly.py       | 4 ++--
 bbot/modules/deadly/ffuf.py            | 6 +-----
 bbot/modules/deadly/nuclei.py          | 4 ++--
 bbot/modules/deadly/vhost.py           | 6 +-----
 bbot/modules/dehashed.py               | 4 ++--
 bbot/modules/digitorus.py              | 4 ++--
 bbot/modules/dnscommonsrv.py           | 6 +-----
 bbot/modules/dnsdumpster.py            | 4 ++--
 bbot/modules/docker_pull.py            | 4 ++--
 bbot/modules/dockerhub.py              | 4 ++--
 bbot/modules/dotnetnuke.py             | 4 ++--
 bbot/modules/emailformat.py            | 4 ++--
 bbot/modules/ffuf_shortnames.py        | 4 ++--
 bbot/modules/filedownload.py           | 4 ++--
 bbot/modules/fingerprintx.py           | 4 ++--
 bbot/modules/fullhunt.py               | 4 ++--
 bbot/modules/generic_ssrf.py           | 6 +-----
 bbot/modules/git.py                    | 4 ++--
 bbot/modules/git_clone.py              | 4 ++--
 bbot/modules/github_codesearch.py      | 4 ++--
 bbot/modules/github_org.py             | 4 ++--
 bbot/modules/github_workflows.py       | 4 ++--
 bbot/modules/gitlab.py                 | 4 ++--
 bbot/modules/gowitness.py              | 6 +-----
 bbot/modules/hackertarget.py           | 4 ++--
 bbot/modules/host_header.py            | 4 ++--
 bbot/modules/httpx.py                  | 4 ++--
 bbot/modules/hunt.py                   | 4 ++--
 bbot/modules/hunterio.py               | 4 ++--
 bbot/modules/iis_shortnames.py         | 4 ++--
 bbot/modules/internal/aggregate.py     | 4 ++--
 bbot/modules/internal/excavate.py      | 4 ++--
 bbot/modules/internal/speculate.py     | 4 ++--
 bbot/modules/internetdb.py             | 4 ++--
 bbot/modules/ip2location.py            | 4 ++--
 bbot/modules/ipneighbor.py             | 4 ++--
 bbot/modules/ipstack.py                | 4 ++--
 bbot/modules/leakix.py                 | 4 ++--
 bbot/modules/masscan.py                | 4 ++--
 bbot/modules/massdns.py                | 4 ++--
 bbot/modules/myssl.py                  | 4 ++--
 bbot/modules/newsletters.py            | 4 ++--
 bbot/modules/nmap.py                   | 4 ++--
 bbot/modules/ntlm.py                   | 4 ++--
 bbot/modules/oauth.py                  | 4 ++--
 bbot/modules/otx.py                    | 4 ++--
 bbot/modules/output/asset_inventory.py | 4 ++--
 bbot/modules/output/csv.py             | 6 +-----
 bbot/modules/output/discord.py         | 4 ++--
 bbot/modules/output/emails.py          | 4 ++--
 bbot/modules/output/http.py            | 4 ++--
 bbot/modules/output/human.py           | 6 +-----
 bbot/modules/output/json.py            | 4 ++--
 bbot/modules/output/neo4j.py           | 6 +-----
 bbot/modules/output/python.py          | 6 +-----
 bbot/modules/output/slack.py           | 4 ++--
 bbot/modules/output/splunk.py          | 4 ++--
 bbot/modules/output/subdomains.py      | 4 ++--
 bbot/modules/output/teams.py           | 4 ++--
 bbot/modules/output/web_report.py      | 4 ++--
 bbot/modules/output/websocket.py       | 6 +-----
 bbot/modules/paramminer_cookies.py     | 2 +-
 bbot/modules/paramminer_getparams.py   | 4 ++--
 bbot/modules/paramminer_headers.py     | 4 ++--
 bbot/modules/passivetotal.py           | 4 ++--
 bbot/modules/pgp.py                    | 4 ++--
 bbot/modules/postman.py                | 4 ++--
 bbot/modules/rapiddns.py               | 4 ++--
 bbot/modules/report/affiliates.py      | 4 ++--
 bbot/modules/report/asn.py             | 4 ++--
 bbot/modules/riddler.py                | 4 ++--
 bbot/modules/robots.py                 | 6 +-----
 bbot/modules/secretsdb.py              | 4 ++--
 bbot/modules/securitytrails.py         | 6 +++---
 bbot/modules/shodan_dns.py             | 4 ++--
 bbot/modules/sitedossier.py            | 4 ++--
 bbot/modules/skymem.py                 | 4 ++--
 bbot/modules/smuggler.py               | 6 +-----
 bbot/modules/social.py                 | 4 ++--
 bbot/modules/sslcert.py                | 2 +-
 bbot/modules/subdomaincenter.py        | 4 ++--
 bbot/modules/sublist3r.py              | 2 +-
 bbot/modules/telerik.py                | 4 ++--
 bbot/modules/threatminer.py            | 2 +-
 bbot/modules/trufflehog.py             | 4 ++--
 bbot/modules/url_manipulation.py       | 4 ++--
 bbot/modules/urlscan.py                | 2 +-
 bbot/modules/viewdns.py                | 2 +-
 bbot/modules/virustotal.py             | 4 ++--
 bbot/modules/wafw00f.py                | 4 ++--
 bbot/modules/wappalyzer.py             | 2 +-
 bbot/modules/wayback.py                | 2 +-
 bbot/modules/zoomeye.py                | 4 ++--
 118 files changed, 214 insertions(+), 270 deletions(-)

diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py
index 019a374bf..dd3abdbf0 100644
--- a/bbot/modules/ajaxpro.py
+++ b/bbot/modules/ajaxpro.py
@@ -14,8 +14,8 @@ class ajaxpro(BaseModule):
     meta = {
         "description": "Check for potentially vulnerable Ajaxpro instances",
         "created_date": "2024-01-18",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     async def handle_event(self, event):
         if event.type == "URL":
diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py
index fa9c21257..88a0a3f26 100644
--- a/bbot/modules/anubisdb.py
+++ b/bbot/modules/anubisdb.py
@@ -8,8 +8,8 @@ class anubisdb(subdomain_enum):
     meta = {
         "description": "Query jldc.me's database for subdomains",
         "created_date": "2022-10-04",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"limit": 1000}
     options_desc = {
         "limit": "Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API)"
diff --git a/bbot/modules/azure_realm.py b/bbot/modules/azure_realm.py
index e65341fa2..7fcbbb5b4 100644
--- a/bbot/modules/azure_realm.py
+++ b/bbot/modules/azure_realm.py
@@ -8,8 +8,8 @@ class azure_realm(BaseModule):
     meta = {
         "description": 'Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm',
         "created_date": "2023-07-12",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     async def setup(self):
         self.processed = set()
diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index bc94480bc..c75271756 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -11,8 +11,8 @@ class azure_tenant(BaseModule):
     meta = {
         "description": "Query Azure for tenant sister domains",
         "created_date": "2024-07-04",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://autodiscover-s.outlook.com"
     in_scope_only = True
diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py
index 987cc0827..d4abbee53 100644
--- a/bbot/modules/baddns.py
+++ b/bbot/modules/baddns.py
@@ -16,8 +16,8 @@ class baddns(BaseModule):
     meta = {
         "description": "Check hosts for domain/subdomain takeovers",
         "created_date": "2024-01-18",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     options = {"custom_nameservers": [], "only_high_confidence": False}
     options_desc = {
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py
index 7b99af1ae..c573078a0 100644
--- a/bbot/modules/baddns_zone.py
+++ b/bbot/modules/baddns_zone.py
@@ -14,8 +14,8 @@ class baddns_zone(baddns_module):
     meta = {
         "description": "Check hosts for DNS zone transfers and NSEC walks",
         "created_date": "2024-01-29",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     options = {"custom_nameservers": [], "only_high_confidence": False}
     options_desc = {
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index e8e4074d4..19b0de720 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -12,8 +12,8 @@ class badsecrets(BaseModule):
     meta = {
         "description": "Library for detecting known or weak secrets across many web frameworks",
         "created_date": "2022-11-19",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     deps_pip = ["badsecrets~=0.4.490"]
 
     @property
diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py
index 85a42c8b0..fa5cf1f1b 100644
--- a/bbot/modules/bevigil.py
+++ b/bbot/modules/bevigil.py
@@ -13,8 +13,8 @@ class bevigil(subdomain_enum_apikey):
         "description": "Retrieve OSINT data from mobile applications using BeVigil",
         "created_date": "2022-10-26",
         "author": "@alt-glitch",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": "", "urls": False}
     options_desc = {"api_key": "BeVigil OSINT API Key", "urls": "Emit URLs in addition to DNS_NAMEs"}
 
diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py
index f65aaf6c6..15d195c01 100644
--- a/bbot/modules/binaryedge.py
+++ b/bbot/modules/binaryedge.py
@@ -9,8 +9,8 @@ class binaryedge(subdomain_enum_apikey):
         "description": "Query the BinaryEdge API",
         "created_date": "2024-08-18",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": "", "max_records": 1000}
     options_desc = {
         "api_key": "BinaryEdge API key",
diff --git a/bbot/modules/bucket_amazon.py b/bbot/modules/bucket_amazon.py
index 93289378c..c61c218fc 100644
--- a/bbot/modules/bucket_amazon.py
+++ b/bbot/modules/bucket_amazon.py
@@ -8,8 +8,8 @@ class bucket_amazon(bucket_template):
     meta = {
         "description": "Check for S3 buckets related to target",
         "created_date": "2022-11-04",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_azure.py b/bbot/modules/bucket_azure.py
index 0261f6ccb..375a565d5 100644
--- a/bbot/modules/bucket_azure.py
+++ b/bbot/modules/bucket_azure.py
@@ -8,8 +8,8 @@ class bucket_azure(bucket_template):
     meta = {
         "description": "Check for Azure storage blobs related to target",
         "created_date": "2022-11-04",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_digitalocean.py b/bbot/modules/bucket_digitalocean.py
index 85bb7c6f3..8e1e008fc 100644
--- a/bbot/modules/bucket_digitalocean.py
+++ b/bbot/modules/bucket_digitalocean.py
@@ -8,8 +8,8 @@ class bucket_digitalocean(bucket_template):
     meta = {
         "description": "Check for DigitalOcean spaces related to target",
         "created_date": "2022-11-08",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py
index 265b1bcff..8b7dcbbb6 100644
--- a/bbot/modules/bucket_file_enum.py
+++ b/bbot/modules/bucket_file_enum.py
@@ -12,7 +12,7 @@ class bucket_file_enum(BaseModule):
     meta = {
         "description": "Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS",
         "created_date": "2023-11-14",
-        "author": "@TheTechromancer"
+        "author": "@TheTechromancer",
     }
     flags = ["passive", "safe", "cloud-enum"]
     options = {
diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py
index db619149f..c3117286e 100644
--- a/bbot/modules/bucket_firebase.py
+++ b/bbot/modules/bucket_firebase.py
@@ -8,8 +8,8 @@ class bucket_firebase(bucket_template):
     meta = {
         "description": "Check for open Firebase databases related to target",
         "created_date": "2023-03-20",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/bucket_google.py b/bbot/modules/bucket_google.py
index 3964072f4..75b41b235 100644
--- a/bbot/modules/bucket_google.py
+++ b/bbot/modules/bucket_google.py
@@ -12,8 +12,8 @@ class bucket_google(bucket_template):
     meta = {
         "description": "Check for Google object storage related to target",
         "created_date": "2022-11-04",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"permutations": False}
     options_desc = {
         "permutations": "Whether to try permutations",
diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py
index bc091122a..7d6197089 100644
--- a/bbot/modules/builtwith.py
+++ b/bbot/modules/builtwith.py
@@ -21,8 +21,8 @@ class builtwith(subdomain_enum_apikey):
         "description": "Query Builtwith.com for subdomains",
         "created_date": "2022-08-23",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": "", "redirects": True}
     options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"}
     base_url = "https://api.builtwith.com"
diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index 0a8de646c..546f52fe2 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -79,11 +79,7 @@ class bypass403(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {
-        "description": "Check 403 pages for common bypasses",
-        "created_date": "2022-07-05",
-        "author": "@liquidsec"
-        }
+    meta = {"description": "Check 403 pages for common bypasses", "created_date": "2022-07-05", "author": "@liquidsec"}
     in_scope_only = True
 
     async def do_checks(self, compare_helper, event, collapse_threshold):
diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py
index 128c0a371..8db5775cb 100644
--- a/bbot/modules/c99.py
+++ b/bbot/modules/c99.py
@@ -9,8 +9,8 @@ class c99(subdomain_enum_apikey):
         "description": "Query the C99 API for subdomains",
         "created_date": "2022-07-08",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "c99.nl API key"}
 
diff --git a/bbot/modules/censys.py b/bbot/modules/censys.py
index 1580a0af8..cb8a7c956 100644
--- a/bbot/modules/censys.py
+++ b/bbot/modules/censys.py
@@ -13,8 +13,8 @@ class censys(subdomain_enum_apikey):
         "description": "Query the Censys API",
         "created_date": "2022-08-04",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_id": "", "api_secret": "", "max_pages": 5}
     options_desc = {
         "api_id": "Censys.io API ID",
diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py
index caafe1898..9d7e00d87 100644
--- a/bbot/modules/certspotter.py
+++ b/bbot/modules/certspotter.py
@@ -8,8 +8,8 @@ class certspotter(subdomain_enum):
     meta = {
         "description": "Query Certspotter's API for subdomains",
         "created_date": "2022-07-28",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://api.certspotter.com/v1"
 
diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py
index 0237a8a20..e98fedd84 100644
--- a/bbot/modules/chaos.py
+++ b/bbot/modules/chaos.py
@@ -9,8 +9,8 @@ class chaos(subdomain_enum_apikey):
         "description": "Query ProjectDiscovery's Chaos API for subdomains",
         "created_date": "2022-08-14",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "Chaos API key"}
 
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
index 4f6ed6049..1a946d13c 100644
--- a/bbot/modules/code_repository.py
+++ b/bbot/modules/code_repository.py
@@ -8,8 +8,8 @@ class code_repository(BaseModule):
     meta = {
         "description": "Look for code repository links in webpages",
         "created_date": "2024-05-15",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
     flags = ["passive", "safe", "repo-enum"]
 
     # platform name : (regex, case_sensitive)
diff --git a/bbot/modules/columbus.py b/bbot/modules/columbus.py
index b4d5370d6..4c09f3a5d 100644
--- a/bbot/modules/columbus.py
+++ b/bbot/modules/columbus.py
@@ -8,8 +8,8 @@ class columbus(subdomain_enum):
     meta = {
         "description": "Query the Columbus Project API for subdomains",
         "created_date": "2023-06-01",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://columbus.elmasy.com/api/lookup"
 
diff --git a/bbot/modules/crobat.py b/bbot/modules/crobat.py
index b5a26c705..f7baacf37 100644
--- a/bbot/modules/crobat.py
+++ b/bbot/modules/crobat.py
@@ -6,9 +6,5 @@ class crobat(subdomain_enum):
     produced_events = ["DNS_NAME"]
     # tag "subdomain-enum" removed 2023-02-24 because API is offline
     flags = ["passive", "safe"]
-    meta = {
-        "description": "Query Project Crobat for subdomains",
-        "created_date": "2022-06-03",
-        "author": "@j3tj3rk"
-        }
+    meta = {"description": "Query Project Crobat for subdomains", "created_date": "2022-06-03", "author": "@j3tj3rk"}
     base_url = "https://sonar.omnisint.io"
diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py
index a2d995a58..27d088a0d 100644
--- a/bbot/modules/crt.py
+++ b/bbot/modules/crt.py
@@ -8,8 +8,8 @@ class crt(subdomain_enum):
     meta = {
         "description": "Query crt.sh (certificate transparency) for subdomains",
         "created_date": "2022-05-13",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://crt.sh"
     reject_wildcards = False
diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py
index d78f1d257..73df81bdf 100644
--- a/bbot/modules/deadly/dastardly.py
+++ b/bbot/modules/deadly/dastardly.py
@@ -9,8 +9,8 @@ class dastardly(BaseModule):
     meta = {
         "description": "Lightweight web application security scanner",
         "created_date": "2023-12-11",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
 
     deps_pip = ["lxml~=4.9.2"]
     deps_ansible = [
diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py
index f08b5d507..ce9dba83f 100644
--- a/bbot/modules/deadly/ffuf.py
+++ b/bbot/modules/deadly/ffuf.py
@@ -11,11 +11,7 @@ class ffuf(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["aggressive", "active"]
-    meta = {
-        "description": "A fast web fuzzer written in Go",
-        "created_date": "2022-04-10",
-        "author": "@pmueller"
-        }
+    meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@pmueller"}
 
     options = {
         "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt",
diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py
index 14e4c1692..af5a7ac53 100644
--- a/bbot/modules/deadly/nuclei.py
+++ b/bbot/modules/deadly/nuclei.py
@@ -11,8 +11,8 @@ class nuclei(BaseModule):
     meta = {
         "description": "Fast and customisable vulnerability scanner",
         "created_date": "2022-03-12",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     options = {
         "version": "3.2.0",
diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py
index ba436ee4e..07fc331bb 100644
--- a/bbot/modules/deadly/vhost.py
+++ b/bbot/modules/deadly/vhost.py
@@ -8,11 +8,7 @@ class vhost(ffuf):
     watched_events = ["URL"]
     produced_events = ["VHOST", "DNS_NAME"]
     flags = ["active", "aggressive", "slow"]
-    meta = {
-        "description": "Fuzz for virtual hosts",
-        "created_date": "2022-05-02",
-        "author": "@liquidsec"
-        }
+    meta = {"description": "Fuzz for virtual hosts", "created_date": "2022-05-02", "author": "@liquidsec"}
 
     special_vhost_list = ["127.0.0.1", "localhost", "host.docker.internal"]
     options = {
diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py
index 5a7016103..89c5d6a17 100644
--- a/bbot/modules/dehashed.py
+++ b/bbot/modules/dehashed.py
@@ -11,8 +11,8 @@ class dehashed(BaseModule):
         "description": "Execute queries against dehashed.com for exposed credentials",
         "created_date": "2023-10-12",
         "author": "@SpamFaux",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"username": "", "api_key": ""}
     options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"}
     target_only = True
diff --git a/bbot/modules/digitorus.py b/bbot/modules/digitorus.py
index b51c0e49f..48c060346 100644
--- a/bbot/modules/digitorus.py
+++ b/bbot/modules/digitorus.py
@@ -10,8 +10,8 @@ class digitorus(subdomain_enum):
     meta = {
         "description": "Query certificatedetails.com for subdomains",
         "created_date": "2023-07-25",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://certificatedetails.com"
 
diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py
index 800015a71..b44c6464f 100644
--- a/bbot/modules/dnscommonsrv.py
+++ b/bbot/modules/dnscommonsrv.py
@@ -153,11 +153,7 @@ class dnscommonsrv(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
-    meta = {
-        "description": "Check for common SRV records",
-        "created_date": "2022-05-15",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Check for common SRV records", "created_date": "2022-05-15", "author": "@TheTechromancer"}
     options = {"top": 50, "max_event_handlers": 10}
     options_desc = {
         "top": "How many of the top SRV records to check",
diff --git a/bbot/modules/dnsdumpster.py b/bbot/modules/dnsdumpster.py
index 40e2dd0de..a1d981565 100644
--- a/bbot/modules/dnsdumpster.py
+++ b/bbot/modules/dnsdumpster.py
@@ -10,8 +10,8 @@ class dnsdumpster(subdomain_enum):
     meta = {
         "description": "Query dnsdumpster for subdomains",
         "created_date": "2022-03-12",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://dnsdumpster.com"
 
diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py
index 80071d2ca..775b5a181 100644
--- a/bbot/modules/docker_pull.py
+++ b/bbot/modules/docker_pull.py
@@ -12,8 +12,8 @@ class docker_pull(BaseModule):
     meta = {
         "description": "Download images from a docker repository",
         "created_date": "2024-03-24",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
     options = {"all_tags": False, "output_folder": ""}
     options_desc = {
         "all_tags": "Download all tags from each registry (Default False)",
diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py
index e2940636a..fcfc189c3 100644
--- a/bbot/modules/dockerhub.py
+++ b/bbot/modules/dockerhub.py
@@ -8,8 +8,8 @@ class dockerhub(BaseModule):
     meta = {
         "description": "Search for docker repositories of discovered orgs/usernames",
         "created_date": "2024-03-12",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
 
     site_url = "https://hub.docker.com"
     api_url = f"{site_url}/v2"
diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index 04588cd0c..2845a5192 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -23,8 +23,8 @@ class dotnetnuke(BaseModule):
     meta = {
         "description": "Scan for critical DotNetNuke (DNN) vulnerabilities",
         "created_date": "2023-11-21",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     async def setup(self):
         self.event_dict = {}
diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py
index 92d98bb29..be797c678 100644
--- a/bbot/modules/emailformat.py
+++ b/bbot/modules/emailformat.py
@@ -8,8 +8,8 @@ class emailformat(BaseModule):
     meta = {
         "description": "Query email-format.com for email addresses",
         "created_date": "2022-07-11",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     in_scope_only = False
     per_domain_only = True
 
diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py
index 3a3d91392..f9e6acd40 100644
--- a/bbot/modules/ffuf_shortnames.py
+++ b/bbot/modules/ffuf_shortnames.py
@@ -36,8 +36,8 @@ class ffuf_shortnames(ffuf):
     meta = {
         "description": "Use ffuf in combination IIS shortnames",
         "created_date": "2022-07-05",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     options = {
         "wordlist": "",  # default is defined within setup function
diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py
index 706fd8c44..98b4d9f01 100644
--- a/bbot/modules/filedownload.py
+++ b/bbot/modules/filedownload.py
@@ -18,8 +18,8 @@ class filedownload(BaseModule):
     meta = {
         "description": "Download common filetypes such as PDF, DOCX, PPTX, etc.",
         "created_date": "2023-10-11",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "extensions": [
             "bak",  #  Backup File
diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py
index ce2c98303..5e69eb3e2 100644
--- a/bbot/modules/fingerprintx.py
+++ b/bbot/modules/fingerprintx.py
@@ -10,8 +10,8 @@ class fingerprintx(BaseModule):
     meta = {
         "description": "Fingerprint exposed services like RDP, SSH, MySQL, etc.",
         "created_date": "2023-01-30",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"version": "1.1.4"}
     options_desc = {"version": "fingerprintx version"}
     _batch_size = 10
diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py
index 08e870bcf..90f83bcfc 100644
--- a/bbot/modules/fullhunt.py
+++ b/bbot/modules/fullhunt.py
@@ -9,8 +9,8 @@ class fullhunt(subdomain_enum_apikey):
         "description": "Query the fullhunt.io API for subdomains",
         "created_date": "2022-08-24",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "FullHunt API Key"}
 
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index 936f09b29..5db762b8c 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -152,11 +152,7 @@ class generic_ssrf(BaseModule):
     watched_events = ["URL"]
     produced_events = ["VULNERABILITY"]
     flags = ["active", "aggressive", "web-thorough"]
-    meta = {
-        "description": "Check for generic SSRFs",
-        "created_date": "2022-07-30",
-        "author": "@liquidsec"
-        }
+    meta = {"description": "Check for generic SSRFs", "created_date": "2022-07-30", "author": "@liquidsec"}
     in_scope_only = True
 
     deps_apt = ["curl"]
diff --git a/bbot/modules/git.py b/bbot/modules/git.py
index 7bedeb650..1fb3235ee 100644
--- a/bbot/modules/git.py
+++ b/bbot/modules/git.py
@@ -10,8 +10,8 @@ class git(BaseModule):
     meta = {
         "description": "Check for exposed .git repositories",
         "created_date": "2023-05-30",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     in_scope_only = True
 
diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py
index bf6ccc1ff..8745d70d8 100644
--- a/bbot/modules/git_clone.py
+++ b/bbot/modules/git_clone.py
@@ -9,8 +9,8 @@ class git_clone(github):
     meta = {
         "description": "Clone code github repositories",
         "created_date": "2024-03-08",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
     options = {"api_key": "", "output_folder": ""}
     options_desc = {"api_key": "Github token", "output_folder": "Folder to clone repositories to"}
 
diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py
index 3952f6758..146cf42e5 100644
--- a/bbot/modules/github_codesearch.py
+++ b/bbot/modules/github_codesearch.py
@@ -9,8 +9,8 @@ class github_codesearch(github):
         "description": "Query Github's API for code containing the target domain name",
         "created_date": "2023-12-14",
         "author": "@domwhewell-sage",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": "", "limit": 100}
     options_desc = {"api_key": "Github token", "limit": "Limit code search to this many results"}
 
diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py
index 4dce3a7ca..ef6ffd6d3 100644
--- a/bbot/modules/github_org.py
+++ b/bbot/modules/github_org.py
@@ -8,8 +8,8 @@ class github_org(github):
     meta = {
         "description": "Query Github's API for organization and member repositories",
         "created_date": "2023-12-14",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
     options = {"api_key": "", "include_members": True, "include_member_repos": False}
     options_desc = {
         "api_key": "Github token",
diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index a18d7462e..bf8f7eb6d 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -11,8 +11,8 @@ class github_workflows(github):
     meta = {
         "description": "Download a github repositories workflow logs",
         "created_date": "2024-04-29",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
     options = {"api_key": "", "num_logs": 1}
     options_desc = {
         "api_key": "Github token",
diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py
index 61a600bf4..7aa91190d 100644
--- a/bbot/modules/gitlab.py
+++ b/bbot/modules/gitlab.py
@@ -8,8 +8,8 @@ class gitlab(BaseModule):
     meta = {
         "description": "Detect GitLab instances and query them for repositories",
         "created_date": "2024-03-11",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "Gitlab access token"}
 
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 4d9397f11..ff20ab9ed 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -10,11 +10,7 @@ class gowitness(BaseModule):
     watched_events = ["URL", "SOCIAL"]
     produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"]
     flags = ["active", "safe", "web-screenshots"]
-    meta = {
-        "description": "Take screenshots of webpages",
-        "created_date": "2022-07-08",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Take screenshots of webpages", "created_date": "2022-07-08", "author": "@TheTechromancer"}
     options = {
         "version": "2.4.2",
         "threads": 4,
diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py
index b4d86ceb4..aee94ccd3 100644
--- a/bbot/modules/hackertarget.py
+++ b/bbot/modules/hackertarget.py
@@ -8,8 +8,8 @@ class hackertarget(subdomain_enum):
     meta = {
         "description": "Query the hackertarget.com API for subdomains",
         "created_date": "2022-07-28",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://api.hackertarget.com"
 
diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py
index d51666f7c..bba8f0aca 100644
--- a/bbot/modules/host_header.py
+++ b/bbot/modules/host_header.py
@@ -9,8 +9,8 @@ class host_header(BaseModule):
     meta = {
         "description": "Try common HTTP Host header spoofing techniques",
         "created_date": "2022-07-27",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     in_scope_only = True
     per_hostport_only = True
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index e11a49c7f..2c2fcfe72 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -14,8 +14,8 @@ class httpx(BaseModule):
     meta = {
         "description": "Visit webpages. Many other modules rely on httpx",
         "created_date": "2022-07-08",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     options = {
         "threads": 50,
diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py
index 8ba6e9260..a86c66cef 100644
--- a/bbot/modules/hunt.py
+++ b/bbot/modules/hunt.py
@@ -278,8 +278,8 @@ class hunt(BaseModule):
     meta = {
         "description": "Watch for commonly-exploitable HTTP parameters",
         "created_date": "2022-07-20",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     # accept all events regardless of scope distance
     scope_distance_modifier = None
 
diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py
index d3f3a27d9..2f4d22bad 100644
--- a/bbot/modules/hunterio.py
+++ b/bbot/modules/hunterio.py
@@ -9,8 +9,8 @@ class hunterio(subdomain_enum_apikey):
         "description": "Query hunter.io for emails",
         "created_date": "2022-04-25",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "Hunter.IO API key"}
 
diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index 56373638e..b25d1ee12 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -20,8 +20,8 @@ class iis_shortnames(BaseModule):
     meta = {
         "description": "Check for IIS shortname vulnerability",
         "created_date": "2022-04-15",
-        "author": "@pmueller"
-        }
+        "author": "@pmueller",
+    }
     options = {"detect_only": True, "max_node_count": 50}
     options_desc = {
         "detect_only": "Only detect the vulnerability and do not run the shortname scanner",
diff --git a/bbot/modules/internal/aggregate.py b/bbot/modules/internal/aggregate.py
index 4a17390b6..54e3a52cc 100644
--- a/bbot/modules/internal/aggregate.py
+++ b/bbot/modules/internal/aggregate.py
@@ -6,8 +6,8 @@ class aggregate(BaseReportModule):
     meta = {
         "description": "Summarize statistics at the end of a scan",
         "created_date": "2022-07-25",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     async def report(self):
         self.log_table(*self.scan.stats._make_table(), table_name="scan-stats")
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index da2f3aa4f..33254d9ef 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -341,8 +341,8 @@ class excavate(BaseInternalModule):
     meta = {
         "description": "Passively extract juicy tidbits from scan data",
         "created_date": "2022-06-27",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     scope_distance_modifier = None
 
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 6aee68a1d..2d7ea7162 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -29,8 +29,8 @@ class speculate(BaseInternalModule):
     meta = {
         "description": "Derive certain event types from others by common sense",
         "created_date": "2022-05-03",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     options = {"max_hosts": 65536, "ports": "80,443"}
     options_desc = {
diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py
index 796299fc1..3003b2e9f 100644
--- a/bbot/modules/internetdb.py
+++ b/bbot/modules/internetdb.py
@@ -41,8 +41,8 @@ class internetdb(BaseModule):
     meta = {
         "description": "Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities",
         "created_date": "2023-12-22",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     _qsize = 500
 
diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py
index be4195bab..6e2bf4a25 100644
--- a/bbot/modules/ip2location.py
+++ b/bbot/modules/ip2location.py
@@ -13,8 +13,8 @@ class IP2Location(BaseModule):
         "description": "Query IP2location.io's API for geolocation information. ",
         "created_date": "2023-09-12",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": "", "lang": ""}
     options_desc = {
         "api_key": "IP2location.io API Key",
diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py
index 9586c707e..e4cc1dd55 100644
--- a/bbot/modules/ipneighbor.py
+++ b/bbot/modules/ipneighbor.py
@@ -10,8 +10,8 @@ class ipneighbor(BaseModule):
     meta = {
         "description": "Look beside IPs in their surrounding subnet",
         "created_date": "2022-06-08",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"num_bits": 4}
     options_desc = {"num_bits": "Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)"}
     scope_distance_modifier = 1
diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py
index 5a6f68759..0d81674a4 100644
--- a/bbot/modules/ipstack.py
+++ b/bbot/modules/ipstack.py
@@ -14,8 +14,8 @@ class Ipstack(BaseModule):
         "description": "Query IPStack's GeoIP API",
         "created_date": "2022-11-26",
         "author": "@tycoonslive",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "IPStack GeoIP API Key"}
     scope_distance_modifier = 1
diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py
index f2e1b1011..4bc88d3c7 100644
--- a/bbot/modules/leakix.py
+++ b/bbot/modules/leakix.py
@@ -11,8 +11,8 @@ class leakix(subdomain_enum_apikey):
     meta = {
         "description": "Query leakix.net for subdomains",
         "created_date": "2022-07-11",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://leakix.net"
 
diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py
index 10e0cd921..502d5dc81 100644
--- a/bbot/modules/masscan.py
+++ b/bbot/modules/masscan.py
@@ -11,8 +11,8 @@ class masscan(portscanner):
     meta = {
         "description": "Port scan with masscan. By default, scans top 100 ports.",
         "created_date": "2023-01-27",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "top_ports": 100,
         "ports": "",
diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py
index 7e3f57a7c..ce172681a 100644
--- a/bbot/modules/massdns.py
+++ b/bbot/modules/massdns.py
@@ -28,8 +28,8 @@ class massdns(subdomain_enum):
     meta = {
         "description": "Brute-force subdomains with massdns (highly effective)",
         "created_date": "2023-03-29",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt",
         "max_resolvers": 1000,
diff --git a/bbot/modules/myssl.py b/bbot/modules/myssl.py
index 5527cfa88..047ced928 100644
--- a/bbot/modules/myssl.py
+++ b/bbot/modules/myssl.py
@@ -8,8 +8,8 @@ class myssl(subdomain_enum):
     meta = {
         "description": "Query myssl.com's API for subdomains",
         "created_date": "2023-07-10",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://myssl.com/api/v1/discover_sub_domain"
 
diff --git a/bbot/modules/newsletters.py b/bbot/modules/newsletters.py
index f8dd33a65..11bb2a8b1 100644
--- a/bbot/modules/newsletters.py
+++ b/bbot/modules/newsletters.py
@@ -22,8 +22,8 @@ class newsletters(BaseModule):
     meta = {
         "description": "Searches for Newsletter Submission Entry Fields on Websites",
         "created_date": "2024-02-02",
-        "author": "@stryker2k2"
-        }
+        "author": "@stryker2k2",
+    }
 
     # Parse through Website to find a Text Entry Box of 'type = email'
     # and ensure that there is placeholder text within it.
diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py
index ec1723ac6..16e7064be 100644
--- a/bbot/modules/nmap.py
+++ b/bbot/modules/nmap.py
@@ -9,8 +9,8 @@ class nmap(portscanner):
     meta = {
         "description": "Port scan with nmap. By default, scans top 100 ports.",
         "created_date": "2022-03-12",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "top_ports": 100,
         "ports": "",
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index 718b800e2..ad963c4b5 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -72,8 +72,8 @@ class ntlm(BaseModule):
     meta = {
         "description": "Watch for HTTP endpoints that support NTLM authentication",
         "created_date": "2022-07-25",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     options = {"try_all": False}
     options_desc = {"try_all": "Try every NTLM endpoint"}
 
diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py
index 5096e3cd4..32418848d 100644
--- a/bbot/modules/oauth.py
+++ b/bbot/modules/oauth.py
@@ -10,8 +10,8 @@ class OAUTH(BaseModule):
     meta = {
         "description": "Enumerate OAUTH and OpenID Connect services",
         "created_date": "2023-07-12",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"try_all": False}
     options_desc = {"try_all": "Check for OAUTH/IODC on every subdomain and URL."}
 
diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py
index bf60420e4..5a2319e3d 100644
--- a/bbot/modules/otx.py
+++ b/bbot/modules/otx.py
@@ -8,8 +8,8 @@ class otx(subdomain_enum):
     meta = {
         "description": "Query otx.alienvault.com for subdomains",
         "created_date": "2022-08-24",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://otx.alienvault.com"
 
diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py
index 4d624588a..82a67e97c 100644
--- a/bbot/modules/output/asset_inventory.py
+++ b/bbot/modules/output/asset_inventory.py
@@ -36,8 +36,8 @@ class asset_inventory(CSV):
     meta = {
         "description": "Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV",
         "created_date": "2022-09-30",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     options = {"output_file": "", "use_previous": False, "recheck": False, "summary_netmask": 16}
     options_desc = {
         "output_file": "Set a custom output file",
diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py
index ae403b34f..3fd28f545 100644
--- a/bbot/modules/output/csv.py
+++ b/bbot/modules/output/csv.py
@@ -6,11 +6,7 @@
 
 class CSV(BaseOutputModule):
     watched_events = ["*"]
-    meta = {
-        "description": "Output to CSV",
-        "created_date": "2022-04-07",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Output to CSV", "created_date": "2022-04-07", "author": "@TheTechromancer"}
     options = {"output_file": ""}
     options_desc = {"output_file": "Output to CSV file"}
 
diff --git a/bbot/modules/output/discord.py b/bbot/modules/output/discord.py
index 8df91eb7d..3a921a900 100644
--- a/bbot/modules/output/discord.py
+++ b/bbot/modules/output/discord.py
@@ -6,8 +6,8 @@ class Discord(WebhookOutputModule):
     meta = {
         "description": "Message a Discord channel when certain events are encountered",
         "created_date": "2023-08-14",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
         "webhook_url": "Discord webhook URL",
diff --git a/bbot/modules/output/emails.py b/bbot/modules/output/emails.py
index 0e13926de..1b0590633 100644
--- a/bbot/modules/output/emails.py
+++ b/bbot/modules/output/emails.py
@@ -8,8 +8,8 @@ class Emails(Human):
     meta = {
         "description": "Output any email addresses found belonging to the target domain",
         "created_date": "2023-12-23",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
     options = {"output_file": ""}
     options_desc = {"output_file": "Output to file"}
     in_scope_only = True
diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py
index 890f55343..e4ed2ddbc 100644
--- a/bbot/modules/output/http.py
+++ b/bbot/modules/output/http.py
@@ -8,8 +8,8 @@ class HTTP(BaseOutputModule):
     meta = {
         "description": "Send every event to a custom URL via a web request",
         "created_date": "2022-04-13",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "url": "",
         "method": "POST",
diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py
index 11c593bee..37b812214 100644
--- a/bbot/modules/output/human.py
+++ b/bbot/modules/output/human.py
@@ -6,11 +6,7 @@
 
 class Human(BaseOutputModule):
     watched_events = ["*"]
-    meta = {
-        "description": "Output to text",
-        "created_date": "2022-04-07",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Output to text", "created_date": "2022-04-07", "author": "@TheTechromancer"}
     options = {"output_file": "", "console": True}
     options_desc = {"output_file": "Output to file", "console": "Output to console"}
     vuln_severity_map = {"LOW": "HUGEWARNING", "MEDIUM": "HUGEWARNING", "HIGH": "CRITICAL", "CRITICAL": "CRITICAL"}
diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py
index 34f45bbc1..e4dcdfd01 100644
--- a/bbot/modules/output/json.py
+++ b/bbot/modules/output/json.py
@@ -9,8 +9,8 @@ class JSON(BaseOutputModule):
     meta = {
         "description": "Output to Newline-Delimited JSON (NDJSON)",
         "created_date": "2022-04-07",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"output_file": "", "console": False, "siem_friendly": False}
     options_desc = {
         "output_file": "Output to file",
diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py
index 711c55d3e..f7ecc3a84 100644
--- a/bbot/modules/output/neo4j.py
+++ b/bbot/modules/output/neo4j.py
@@ -25,11 +25,7 @@ class neo4j(BaseOutputModule):
     """
 
     watched_events = ["*"]
-    meta = {
-        "description": "Output to Neo4j",
-        "created_date": "2022-04-07",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Output to Neo4j", "created_date": "2022-04-07", "author": "@TheTechromancer"}
     options = {"uri": "bolt://localhost:7687", "username": "neo4j", "password": "bbotislife"}
     options_desc = {
         "uri": "Neo4j server + port",
diff --git a/bbot/modules/output/python.py b/bbot/modules/output/python.py
index 27740a9bb..81ceb360e 100644
--- a/bbot/modules/output/python.py
+++ b/bbot/modules/output/python.py
@@ -3,11 +3,7 @@
 
 class python(BaseOutputModule):
     watched_events = ["*"]
-    meta = {
-        "description": "Output via Python API",
-        "created_date": "2022-09-13",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Output via Python API", "created_date": "2022-09-13", "author": "@TheTechromancer"}
 
     async def _worker(self):
         pass
diff --git a/bbot/modules/output/slack.py b/bbot/modules/output/slack.py
index fea4326a1..438ef4973 100644
--- a/bbot/modules/output/slack.py
+++ b/bbot/modules/output/slack.py
@@ -8,8 +8,8 @@ class Slack(WebhookOutputModule):
     meta = {
         "description": "Message a Slack channel when certain events are encountered",
         "created_date": "2023-08-14",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
         "webhook_url": "Discord webhook URL",
diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py
index 3a82d1d99..de3e8d131 100644
--- a/bbot/modules/output/splunk.py
+++ b/bbot/modules/output/splunk.py
@@ -8,8 +8,8 @@ class Splunk(BaseOutputModule):
     meta = {
         "description": "Send every event to a splunk instance through HTTP Event Collector",
         "created_date": "2024-02-17",
-        "author": "@w0Tx"
-        }
+        "author": "@w0Tx",
+    }
     options = {
         "url": "",
         "hectoken": "",
diff --git a/bbot/modules/output/subdomains.py b/bbot/modules/output/subdomains.py
index 779f806dc..e8c0ec5a3 100644
--- a/bbot/modules/output/subdomains.py
+++ b/bbot/modules/output/subdomains.py
@@ -8,8 +8,8 @@ class Subdomains(Human):
     meta = {
         "description": "Output only resolved, in-scope subdomains",
         "created_date": "2023-07-31",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"output_file": "", "include_unresolved": False}
     options_desc = {"output_file": "Output to file", "include_unresolved": "Include unresolved subdomains in output"}
     accept_dupes = False
diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py
index 8659a9eed..e4f87fb71 100644
--- a/bbot/modules/output/teams.py
+++ b/bbot/modules/output/teams.py
@@ -6,8 +6,8 @@ class Teams(WebhookOutputModule):
     meta = {
         "description": "Message a Teams channel when certain events are encountered",
         "created_date": "2023-08-14",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {"webhook_url": "", "event_types": ["VULNERABILITY", "FINDING"], "min_severity": "LOW"}
     options_desc = {
         "webhook_url": "Discord webhook URL",
diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py
index e4086a449..ba69f9b2a 100644
--- a/bbot/modules/output/web_report.py
+++ b/bbot/modules/output/web_report.py
@@ -8,8 +8,8 @@ class web_report(BaseOutputModule):
     meta = {
         "description": "Create a markdown report with web assets",
         "created_date": "2023-02-08",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     options = {
         "output_file": "",
         "css_theme_file": "https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css",
diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py
index f3ef57a85..c8f54097a 100644
--- a/bbot/modules/output/websocket.py
+++ b/bbot/modules/output/websocket.py
@@ -7,11 +7,7 @@
 
 class Websocket(BaseOutputModule):
     watched_events = ["*"]
-    meta = {
-        "description": "Output to websockets",
-        "created_date": "2022-04-15",
-        "author": "@TheTechromancer"
-        }
+    meta = {"description": "Output to websockets", "created_date": "2022-04-15", "author": "@TheTechromancer"}
     options = {"url": "", "token": "", "preserve_graph": True}
     options_desc = {
         "url": "Web URL",
diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py
index ea671929a..6a113f7f7 100644
--- a/bbot/modules/paramminer_cookies.py
+++ b/bbot/modules/paramminer_cookies.py
@@ -12,7 +12,7 @@ class paramminer_cookies(paramminer_headers):
     meta = {
         "description": "Smart brute-force to check for common HTTP cookie parameters",
         "created_date": "2022-06-27",
-        "author": "@liquidsec"
+        "author": "@liquidsec",
     }
     options = {
         "wordlist": "",  # default is defined within setup function
diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py
index e4fdc2949..c596dd0ec 100644
--- a/bbot/modules/paramminer_getparams.py
+++ b/bbot/modules/paramminer_getparams.py
@@ -12,8 +12,8 @@ class paramminer_getparams(paramminer_headers):
     meta = {
         "description": "Use smart brute-force to check for common HTTP GET parameters",
         "created_date": "2022-06-28",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     scanned_hosts = []
     options = {
         "wordlist": "",  # default is defined within setup function
diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index 6979e1330..5aac4b304 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -14,8 +14,8 @@ class paramminer_headers(BaseModule):
     meta = {
         "description": "Use smart brute-force to check for common HTTP header parameters",
         "created_date": "2022-04-15",
-        "author": "@pmueller"
-        }
+        "author": "@pmueller",
+    }
     options = {
         "wordlist": "",  # default is defined within setup function
         "http_extract": True,
diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py
index 999085baa..e22dfc3e5 100644
--- a/bbot/modules/passivetotal.py
+++ b/bbot/modules/passivetotal.py
@@ -9,8 +9,8 @@ class passivetotal(subdomain_enum_apikey):
         "description": "Query the PassiveTotal API for subdomains",
         "created_date": "2022-08-08",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"username": "", "api_key": ""}
     options_desc = {"username": "RiskIQ Username", "api_key": "RiskIQ API Key"}
 
diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py
index 5b52bbd14..54157930a 100644
--- a/bbot/modules/pgp.py
+++ b/bbot/modules/pgp.py
@@ -8,8 +8,8 @@ class pgp(subdomain_enum):
     meta = {
         "description": "Query common PGP servers for email addresses",
         "created_date": "2022-08-10",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "search_urls": [
             "https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=<query>",
diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py
index feff85c1f..bfdc2f90d 100644
--- a/bbot/modules/postman.py
+++ b/bbot/modules/postman.py
@@ -8,8 +8,8 @@ class postman(subdomain_enum):
     meta = {
         "description": "Query Postman's API for related workspaces, collections, requests",
         "created_date": "2023-12-23",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
 
     base_url = "https://www.postman.com/_api"
 
diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py
index 05b807773..55cca4dfe 100644
--- a/bbot/modules/rapiddns.py
+++ b/bbot/modules/rapiddns.py
@@ -8,8 +8,8 @@ class rapiddns(subdomain_enum):
     meta = {
         "description": "Query rapiddns.io for subdomains",
         "created_date": "2022-08-24",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://rapiddns.io"
 
diff --git a/bbot/modules/report/affiliates.py b/bbot/modules/report/affiliates.py
index 56357bee0..73b2867e2 100644
--- a/bbot/modules/report/affiliates.py
+++ b/bbot/modules/report/affiliates.py
@@ -8,8 +8,8 @@ class affiliates(BaseReportModule):
     meta = {
         "description": "Summarize affiliate domains at the end of a scan",
         "created_date": "2022-07-25",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     scope_distance_modifier = None
     accept_dupes = True
 
diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py
index 4802c5659..a89e04fbf 100644
--- a/bbot/modules/report/asn.py
+++ b/bbot/modules/report/asn.py
@@ -8,8 +8,8 @@ class asn(BaseReportModule):
     meta = {
         "description": "Query ripe and bgpview.io for ASNs",
         "created_date": "2022-07-25",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     scope_distance_modifier = 1
     # we accept dupes to avoid missing data
     # because sometimes IP addresses are re-emitted with lower scope distances
diff --git a/bbot/modules/riddler.py b/bbot/modules/riddler.py
index a5c2badac..dbc86e6e0 100644
--- a/bbot/modules/riddler.py
+++ b/bbot/modules/riddler.py
@@ -8,8 +8,8 @@ class riddler(subdomain_enum):
     meta = {
         "description": "Query riddler.io for subdomains",
         "created_date": "2022-08-25",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://riddler.io"
 
diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py
index 57d65503c..8faf59664 100644
--- a/bbot/modules/robots.py
+++ b/bbot/modules/robots.py
@@ -5,11 +5,7 @@ class robots(BaseModule):
     watched_events = ["URL"]
     produced_events = ["URL_UNVERIFIED"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
-    meta = {
-        "description": "Look for and parse robots.txt",
-        "created_date": "2023-02-01",
-        "author": "@liquidsec"
-        }
+    meta = {"description": "Look for and parse robots.txt", "created_date": "2023-02-01", "author": "@liquidsec"}
 
     options = {"include_sitemap": False, "include_allow": True, "include_disallow": True}
     options_desc = {
diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py
index 9facf5ff3..62246e1cc 100644
--- a/bbot/modules/secretsdb.py
+++ b/bbot/modules/secretsdb.py
@@ -11,8 +11,8 @@ class secretsdb(BaseModule):
     meta = {
         "description": "Detect common secrets with secrets-patterns-db",
         "created_date": "2023-03-17",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     options = {
         "min_confidence": 99,
         "signatures": "https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml",
diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py
index 7899b55ee..a91db2912 100644
--- a/bbot/modules/securitytrails.py
+++ b/bbot/modules/securitytrails.py
@@ -8,9 +8,9 @@ class securitytrails(subdomain_enum_apikey):
     meta = {
         "description": "Query the SecurityTrails API for subdomains",
         "created_date": "2022-07-03",
-        "author": "@TheTechromancer"
-        "auth_required": True
-        }
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "SecurityTrails API key"}
 
diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py
index a4248cbdc..2336e0243 100644
--- a/bbot/modules/shodan_dns.py
+++ b/bbot/modules/shodan_dns.py
@@ -9,8 +9,8 @@ class shodan_dns(shodan):
         "description": "Query Shodan for subdomains",
         "created_date": "2022-07-03",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "Shodan API key"}
 
diff --git a/bbot/modules/sitedossier.py b/bbot/modules/sitedossier.py
index 897201ac3..da11ac82f 100644
--- a/bbot/modules/sitedossier.py
+++ b/bbot/modules/sitedossier.py
@@ -8,8 +8,8 @@ class sitedossier(subdomain_enum):
     meta = {
         "description": "Query sitedossier.com for subdomains",
         "created_date": "2023-08-04",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "http://www.sitedossier.com/parentdomain"
     max_pages = 10
diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py
index 371cd39d3..24f8ce309 100644
--- a/bbot/modules/skymem.py
+++ b/bbot/modules/skymem.py
@@ -10,8 +10,8 @@ class skymem(emailformat):
     meta = {
         "description": "Query skymem.info for email addresses",
         "created_date": "2022-07-11",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://www.skymem.info"
 
diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py
index b921894f2..6a3c5a85a 100644
--- a/bbot/modules/smuggler.py
+++ b/bbot/modules/smuggler.py
@@ -12,11 +12,7 @@ class smuggler(BaseModule):
     watched_events = ["URL"]
     produced_events = ["FINDING"]
     flags = ["active", "aggressive", "slow", "web-thorough"]
-    meta = {
-        "description": "Check for HTTP smuggling",
-        "created_date": "2022-07-06",
-        "author": "@liquidsec"
-        }
+    meta = {"description": "Check for HTTP smuggling", "created_date": "2022-07-06", "author": "@liquidsec"}
 
     in_scope_only = True
     per_hostport_only = True
diff --git a/bbot/modules/social.py b/bbot/modules/social.py
index fdb106e8b..4dbf05948 100644
--- a/bbot/modules/social.py
+++ b/bbot/modules/social.py
@@ -8,8 +8,8 @@ class social(BaseModule):
     meta = {
         "description": "Look for social media links in webpages",
         "created_date": "2023-03-28",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
     flags = ["passive", "safe", "social-enum"]
 
     # platform name : (regex, case_sensitive)
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 03cf3b300..12339af7b 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -14,7 +14,7 @@ class sslcert(BaseModule):
     meta = {
         "description": "Visit open ports and retrieve SSL certificates",
         "created_date": "2022-03-30",
-        "author": "@TheTechromancer"
+        "author": "@TheTechromancer",
     }
     options = {"timeout": 5.0, "skip_non_ssl": True}
     options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"}
diff --git a/bbot/modules/subdomaincenter.py b/bbot/modules/subdomaincenter.py
index 85f0ecbd7..9fdce8c49 100644
--- a/bbot/modules/subdomaincenter.py
+++ b/bbot/modules/subdomaincenter.py
@@ -8,8 +8,8 @@ class subdomaincenter(subdomain_enum):
     meta = {
         "description": "Query subdomain.center's API for subdomains",
         "created_date": "2023-07-26",
-        "author": "@TheTechromancer"
-        }
+        "author": "@TheTechromancer",
+    }
 
     base_url = "https://api.subdomain.center"
     retries = 2
diff --git a/bbot/modules/sublist3r.py b/bbot/modules/sublist3r.py
index 0a673bad7..502352bc9 100644
--- a/bbot/modules/sublist3r.py
+++ b/bbot/modules/sublist3r.py
@@ -9,7 +9,7 @@ class sublist3r(subdomain_enum):
     meta = {
         "description": "Query sublist3r's API for subdomains",
         "created_date": "2022-03-29",
-        "author": "@Want-EyeTea"
+        "author": "@Want-EyeTea",
     }
 
     base_url = "https://api.sublist3r.com/search.php"
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index b70a3b054..64a6437c3 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -12,8 +12,8 @@ class telerik(BaseModule):
     meta = {
         "description": "Scan for critical Telerik vulnerabilities",
         "created_date": "2022-04-10",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     telerikVersions = [
         "2007.1423",
diff --git a/bbot/modules/threatminer.py b/bbot/modules/threatminer.py
index c3e5cd958..b4ccad4b8 100644
--- a/bbot/modules/threatminer.py
+++ b/bbot/modules/threatminer.py
@@ -8,7 +8,7 @@ class threatminer(subdomain_enum):
     meta = {
         "description": "Query threatminer's API for subdomains",
         "created_date": "2022-07-28",
-        "author": "@TheTechromancer"
+        "author": "@TheTechromancer",
     }
 
     base_url = "https://api.threatminer.org/v2"
diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index 8fb05c47f..e5f8dd26d 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -9,8 +9,8 @@ class trufflehog(BaseModule):
     meta = {
         "description": "TruffleHog is a tool for finding credentials",
         "created_date": "2024-03-12",
-        "author": "@domwhewell-sage"
-        }
+        "author": "@domwhewell-sage",
+    }
 
     options = {
         "version": "3.75.1",
diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py
index 9f17c8157..209e51a11 100644
--- a/bbot/modules/url_manipulation.py
+++ b/bbot/modules/url_manipulation.py
@@ -9,8 +9,8 @@ class url_manipulation(BaseModule):
     meta = {
         "description": "Attempt to identify URL parsing/routing based vulnerabilities",
         "created_date": "2022-09-27",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
     in_scope_only = True
 
     options = {"allow_redirects": True}
diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py
index 49b6ca0ee..d9e6cbb43 100644
--- a/bbot/modules/urlscan.py
+++ b/bbot/modules/urlscan.py
@@ -8,7 +8,7 @@ class urlscan(subdomain_enum):
     meta = {
         "description": "Query urlscan.io for subdomains",
         "created_date": "2022-06-09",
-        "author": "@TheTechromancer"
+        "author": "@TheTechromancer",
     }
     options = {"urls": False}
     options_desc = {"urls": "Emit URLs in addition to DNS_NAMEs"}
diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py
index 6f382bf2a..2c1ea73bf 100644
--- a/bbot/modules/viewdns.py
+++ b/bbot/modules/viewdns.py
@@ -14,7 +14,7 @@ class viewdns(BaseModule):
     meta = {
         "description": "Query viewdns.info's reverse whois for related domains",
         "created_date": "2022-07-04",
-        "author": "@TheTechromancer"
+        "author": "@TheTechromancer",
     }
     base_url = "https://viewdns.info"
     in_scope_only = True
diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py
index 323c4cfb8..6f0ba5e82 100644
--- a/bbot/modules/virustotal.py
+++ b/bbot/modules/virustotal.py
@@ -9,8 +9,8 @@ class virustotal(subdomain_enum_apikey):
         "description": "Query VirusTotal's API for subdomains",
         "created_date": "2022-08-25",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": ""}
     options_desc = {"api_key": "VirusTotal API Key"}
 
diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py
index 5af886aeb..c8f9c4b33 100644
--- a/bbot/modules/wafw00f.py
+++ b/bbot/modules/wafw00f.py
@@ -19,8 +19,8 @@ class wafw00f(BaseModule):
     meta = {
         "description": "Web Application Firewall Fingerprinting Tool",
         "created_date": "2023-02-15",
-        "author": "@liquidsec"
-        }
+        "author": "@liquidsec",
+    }
 
     deps_pip = ["wafw00f~=2.2.0"]
 
diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py
index 61a858b9f..ee65e22fe 100644
--- a/bbot/modules/wappalyzer.py
+++ b/bbot/modules/wappalyzer.py
@@ -17,7 +17,7 @@ class wappalyzer(BaseModule):
     meta = {
         "description": "Extract technologies from web responses",
         "created_date": "2022-04-15",
-        "author": "@liquidsec"
+        "author": "@liquidsec",
     }
     deps_pip = ["python-Wappalyzer~=0.3.1", "aiohttp~=3.9.0b0"]
     # accept all events regardless of scope distance
diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py
index 75f45d405..0cf34ce2b 100644
--- a/bbot/modules/wayback.py
+++ b/bbot/modules/wayback.py
@@ -10,7 +10,7 @@ class wayback(subdomain_enum):
     meta = {
         "description": "Query archive.org's API for subdomains",
         "created_date": "2022-04-01",
-        "author": "@pmueller"
+        "author": "@pmueller",
     }
     options = {"urls": False, "garbage_threshold": 10}
     options_desc = {
diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py
index 75823d308..98397729a 100644
--- a/bbot/modules/zoomeye.py
+++ b/bbot/modules/zoomeye.py
@@ -9,8 +9,8 @@ class zoomeye(subdomain_enum_apikey):
         "description": "Query ZoomEye's API for subdomains",
         "created_date": "2022-08-03",
         "author": "@TheTechromancer",
-        "auth_required": True
-        }
+        "auth_required": True,
+    }
     options = {"api_key": "", "max_pages": 20, "include_related": False}
     options_desc = {
         "api_key": "ZoomEye API key",

From 865d96c6601ca9a4b0d2734ed4a04d9a1bf3ae90 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 22 May 2024 15:46:02 -0400
Subject: [PATCH 172/220] remove deleted modules

---
 bbot/modules/massdns.py      | 485 -----------------------------------
 bbot/modules/output/human.py |  49 ----
 2 files changed, 534 deletions(-)
 delete mode 100644 bbot/modules/massdns.py
 delete mode 100644 bbot/modules/output/human.py

diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py
deleted file mode 100644
index ce172681a..000000000
--- a/bbot/modules/massdns.py
+++ /dev/null
@@ -1,485 +0,0 @@
-import re
-import json
-import random
-import asyncio
-import subprocess
-
-from bbot.modules.templates.subdomain_enum import subdomain_enum
-
-
-class massdns(subdomain_enum):
-    """
-    This is BBOT's flagship subdomain enumeration module.
-
-    It uses massdns to brute-force subdomains.
-    At the end of a scan, it will leverage BBOT's word cloud to recursively discover target-specific subdomain mutations.
-
-    Each subdomain discovered via mutations is tagged with the "mutation" tag. This tag indicates the depth at which
-    the mutation was found. I.e. the first mutation will be tagged "mutation-1". The second one (a mutation of a
-    mutation) will be "mutation-2". Mutations of mutations of mutations will be "mutation-3", etc.
-
-    This is especially use for bug bounties because it enables you to recognize distant/rare subdomains at a glance.
-    Subdomains with higher mutation levels are more likely to be distant/rare or never-before-seen.
-    """
-
-    flags = ["subdomain-enum", "passive", "aggressive"]
-    watched_events = ["DNS_NAME"]
-    produced_events = ["DNS_NAME"]
-    meta = {
-        "description": "Brute-force subdomains with massdns (highly effective)",
-        "created_date": "2023-03-29",
-        "author": "@TheTechromancer",
-    }
-    options = {
-        "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt",
-        "max_resolvers": 1000,
-        "max_mutations": 500,
-        "max_depth": 5,
-    }
-    options_desc = {
-        "wordlist": "Subdomain wordlist URL",
-        "max_resolvers": "Number of concurrent massdns resolvers",
-        "max_mutations": "Max number of smart mutations per subdomain",
-        "max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com",
-    }
-    subdomain_file = None
-    deps_ansible = [
-        {
-            "name": "install dev tools",
-            "package": {"name": ["gcc", "git", "make"], "state": "present"},
-            "become": True,
-            "ignore_errors": True,
-        },
-        {
-            "name": "Download massdns source code",
-            "git": {
-                "repo": "https://github.com/blechschmidt/massdns.git",
-                "dest": "#{BBOT_TEMP}/massdns",
-                "single_branch": True,
-                "version": "master",
-            },
-        },
-        {
-            "name": "Build massdns (Linux)",
-            "command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"},
-            "when": "ansible_facts['system'] == 'Linux'",
-        },
-        {
-            "name": "Build massdns (non-Linux)",
-            "command": {
-                "chdir": "#{BBOT_TEMP}/massdns",
-                "cmd": "make nolinux",
-                "creates": "#{BBOT_TEMP}/massdns/bin/massdns",
-            },
-            "when": "ansible_facts['system'] != 'Linux'",
-        },
-        {
-            "name": "Install massdns",
-            "copy": {"src": "#{BBOT_TEMP}/massdns/bin/massdns", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
-        },
-    ]
-    reject_wildcards = "strict"
-    _qsize = 10000
-
-    digit_regex = re.compile(r"\d+")
-
-    async def setup(self):
-        self.found = dict()
-        self.mutations_tried = set()
-        self.source_events = self.helpers.make_target()
-        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
-        self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
-
-        ms_on_prem_string_file = self.helpers.wordlist_dir / "ms_on_prem_subdomains.txt"
-        ms_on_prem_strings = set(self.helpers.read_file(ms_on_prem_string_file))
-        self.subdomain_list.update(ms_on_prem_strings)
-
-        self.max_resolvers = self.config.get("max_resolvers", 1000)
-        self.max_mutations = self.config.get("max_mutations", 500)
-        self.max_depth = max(1, self.config.get("max_depth", 5))
-        nameservers_url = (
-            "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
-        )
-        self.resolver_file = await self.helpers.wordlist(
-            nameservers_url,
-            cache_hrs=24 * 7,
-        )
-        self.devops_mutations = list(self.helpers.word_cloud.devops_mutations)
-        self.mutation_run = 1
-
-        self.resolve_and_emit_queue = asyncio.Queue()
-        self.resolve_and_emit_task = asyncio.create_task(self.resolve_and_emit())
-        return await super().setup()
-
-    async def filter_event(self, event):
-        query = self.make_query(event)
-        eligible, reason = await self.eligible_for_enumeration(event)
-
-        # limit brute force depth
-        subdomain_depth = self.helpers.subdomain_depth(query) + 1
-        if subdomain_depth > self.max_depth:
-            eligible = False
-            reason = f"subdomain depth of *.{query} ({subdomain_depth}) > max_depth ({self.max_depth})"
-
-        # don't brute-force things that look like autogenerated PTRs
-        if self.helpers.is_ptr(query):
-            eligible = False
-            reason = f'"{query}" looks like an autogenerated PTR'
-
-        if eligible:
-            self.add_found(event)
-        # reject if already processed
-        if self.already_processed(query):
-            return False, f'Query "{query}" was already processed'
-
-        if eligible:
-            self.processed.add(hash(query))
-            return True, reason
-        return False, reason
-
-    async def handle_event(self, event):
-        query = self.make_query(event)
-        self.source_events.add_target(event)
-        self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
-        results = await self.massdns(query, self.subdomain_list)
-        await self.resolve_and_emit_queue.put((results, event, None))
-
-    def abort_if(self, event):
-        if not event.scope_distance == 0:
-            return True, "event is not in scope"
-        if "wildcard" in event.tags:
-            return True, "event is a wildcard"
-        if "unresolved" in event.tags:
-            return True, "event is unresolved"
-        return False, ""
-
-    def already_processed(self, hostname):
-        if hash(hostname) in self.processed:
-            return True
-        return False
-
-    async def massdns(self, domain, subdomains):
-        subdomains = list(subdomains)
-
-        domain_wildcard_rdtypes = set()
-        for _domain, rdtypes in (await self.helpers.is_wildcard_domain(domain)).items():
-            for rdtype, results in rdtypes.items():
-                if results:
-                    domain_wildcard_rdtypes.add(rdtype)
-        if any([r in domain_wildcard_rdtypes for r in ("A", "CNAME")]):
-            self.info(
-                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
-            )
-            self.found.pop(domain, None)
-            return []
-        else:
-            self.log.trace(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
-
-        # before we start, do a canary check for wildcards
-        abort_msg = f"Aborting massdns on {domain} due to false positive"
-        canary_result = await self._canary_check(domain)
-        if canary_result:
-            self.info(abort_msg + f": {canary_result}")
-            return []
-        else:
-            self.log.trace(f"Canary result for {domain}: {canary_result}")
-
-        results = []
-        async for hostname, ip, rdtype in self._massdns(domain, subdomains):
-            # allow brute-forcing of wildcard domains
-            # this is dead code but it's kinda cool so it can live here
-            if rdtype in domain_wildcard_rdtypes:
-                # skip wildcard checking on multi-level subdomains for performance reasons
-                stem = hostname.split(domain)[0].strip(".")
-                if "." in stem:
-                    self.debug(f"Skipping {hostname}:A because it may be a wildcard (reason: performance)")
-                    continue
-                wildcard_rdtypes = await self.helpers.is_wildcard(hostname, ips=(ip,), rdtype=rdtype)
-                if rdtype in wildcard_rdtypes:
-                    self.debug(f"Skipping {hostname}:{rdtype} because it's a wildcard")
-                    continue
-            results.append(hostname)
-
-        # do another canary check for good measure
-        if len(results) > 50:
-            canary_result = await self._canary_check(domain)
-            if canary_result:
-                self.info(abort_msg + f": {canary_result}")
-                return []
-            else:
-                self.log.trace(f"Canary result for {domain}: {canary_result}")
-
-        # abort if there are a suspiciously high number of results
-        # (the results are over 2000, and this is more than 20 percent of the input size)
-        if len(results) > 2000:
-            if len(results) / len(subdomains) > 0.2:
-                self.info(
-                    f"Aborting because the number of results ({len(results):,}) is suspiciously high for the length of the wordlist ({len(subdomains):,})"
-                )
-                return []
-            else:
-                self.info(
-                    f"{len(results):,} results returned from massdns against {domain} (wordlist size = {len(subdomains):,})"
-                )
-
-        # everything checks out
-        return results
-
-    async def resolve_and_emit(self):
-        """
-        When results are found, they are placed into self.resolve_and_emit_queue.
-        The purpose of this function (which is started as a task in the module's setup()) is to consume results from
-        the queue, resolve them, and if they resolve, emit them.
-
-        This exists to prevent disrupting the scan with huge batches of DNS resolutions.
-        """
-        while 1:
-            results, source_event, tags = await self.resolve_and_emit_queue.get()
-            self.verbose(f"Resolving batch of {len(results):,} results")
-            async with self._task_counter.count(f"{self.name}.resolve_and_emit()"):
-                async for hostname, r in self.helpers.resolve_batch(results, type=("A", "CNAME")):
-                    if not r:
-                        self.debug(f"Discarding {hostname} because it didn't resolve")
-                        continue
-                    self.add_found(hostname)
-                    if source_event is None:
-                        source_event = self.source_events.get(hostname)
-                        if source_event is None:
-                            self.warning(f"Could not correlate source event from: {hostname}")
-                            source_event = self.scan.root_event
-                    kwargs = {"abort_if": self.abort_if, "tags": tags}
-                    await self.emit_event(hostname, "DNS_NAME", source_event, **kwargs)
-
-    @property
-    def running(self):
-        return super().running or self.resolve_and_emit_queue.qsize() > 0
-
-    async def _canary_check(self, domain, num_checks=50):
-        random_subdomains = list(self.gen_random_subdomains(num_checks))
-        self.verbose(f"Testing {len(random_subdomains):,} canaries against {domain}")
-        canary_results = [h async for h, d, r in self._massdns(domain, random_subdomains)]
-        self.log.trace(f"canary results for {domain}: {canary_results}")
-        resolved_canaries = self.helpers.resolve_batch(canary_results)
-        self.log.trace(f"resolved canary results for {domain}: {canary_results}")
-        async for query, result in resolved_canaries:
-            if result:
-                await resolved_canaries.aclose()
-                result = f"{query}:{result}"
-                self.log.trace(f"Found false positive: {result}")
-                return result
-        self.log.trace(f"Passed canary check for {domain}")
-        return False
-
-    async def _massdns(self, domain, subdomains):
-        """
-        {
-          "name": "www.blacklanternsecurity.com.",
-          "type": "A",
-          "class": "IN",
-          "status": "NOERROR",
-          "data": {
-            "answers": [
-              {
-                "ttl": 3600,
-                "type": "CNAME",
-                "class": "IN",
-                "name": "www.blacklanternsecurity.com.",
-                "data": "blacklanternsecurity.github.io."
-              },
-              {
-                "ttl": 3600,
-                "type": "A",
-                "class": "IN",
-                "name": "blacklanternsecurity.github.io.",
-                "data": "185.199.108.153"
-              }
-            ]
-          },
-          "resolver": "168.215.165.186:53"
-        }
-        """
-        if self.scan.stopping:
-            return
-
-        command = (
-            "massdns",
-            "-r",
-            self.resolver_file,
-            "-s",
-            self.max_resolvers,
-            "-t",
-            "A",
-            "-o",
-            "J",
-            "-q",
-        )
-        subdomains = self.gen_subdomains(subdomains, domain)
-        hosts_yielded = set()
-        async for line in self.run_process_live(command, stderr=subprocess.DEVNULL, input=subdomains):
-            try:
-                j = json.loads(line)
-            except json.decoder.JSONDecodeError:
-                self.debug(f"Failed to decode line: {line}")
-                continue
-            answers = j.get("data", {}).get("answers", [])
-            if type(answers) == list and len(answers) > 0:
-                answer = answers[0]
-                hostname = answer.get("name", "").strip(".").lower()
-                if hostname.endswith(f".{domain}"):
-                    data = answer.get("data", "")
-                    rdtype = answer.get("type", "").upper()
-                    # avoid garbage answers like this:
-                    # 8AAAA queries have been locally blocked by dnscrypt-proxy/Set block_ipv6 to false to disable this feature
-                    if data and rdtype and not " " in data:
-                        hostname_hash = hash(hostname)
-                        if hostname_hash not in hosts_yielded:
-                            hosts_yielded.add(hostname_hash)
-                            yield hostname, data, rdtype
-
-    async def finish(self):
-        found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True)
-        # if we have a lot of rounds to make, don't try mutations on less-populated domains
-        trimmed_found = []
-        if found:
-            avg_subdomains = sum([len(subdomains) for domain, subdomains in found[:50]]) / len(found[:50])
-            for i, (domain, subdomains) in enumerate(found):
-                # accept domains that are in the top 50 or have more than 5 percent of the average number of subdomains
-                if i < 50 or (len(subdomains) > 1 and len(subdomains) >= (avg_subdomains * 0.05)):
-                    trimmed_found.append((domain, subdomains))
-                else:
-                    self.verbose(
-                        f"Skipping mutations on {domain} because it only has {len(subdomains):,} subdomain(s) (avg: {avg_subdomains:,})"
-                    )
-
-        base_mutations = set()
-        found_mutations = False
-        try:
-            for i, (domain, subdomains) in enumerate(trimmed_found):
-                self.verbose(f"{domain} has {len(subdomains):,} subdomains")
-                # keep looping as long as we're finding things
-                while 1:
-                    max_mem_percent = 90
-                    mem_status = self.helpers.memory_status()
-                    # abort if we don't have the memory
-                    mem_percent = mem_status.percent
-                    if mem_percent > max_mem_percent:
-                        free_memory = mem_status.available
-                        free_memory_human = self.helpers.bytes_to_human(free_memory)
-                        assert (
-                            False
-                        ), f"Cannot proceed with DNS mutations because system memory is at {mem_percent:.1f}% ({free_memory_human} remaining)"
-
-                    query = domain
-                    domain_hash = hash(domain)
-                    if self.scan.stopping:
-                        return
-
-                    mutations = set(base_mutations)
-
-                    def add_mutation(_domain_hash, m):
-                        h = hash((_domain_hash, m))
-                        if h not in self.mutations_tried:
-                            self.mutations_tried.add(h)
-                            mutations.add(m)
-
-                    num_base_mutations = len(base_mutations)
-                    self.debug(f"Base mutations for {domain}: {num_base_mutations:,}")
-
-                    # try every subdomain everywhere else
-                    for _domain, _subdomains in found:
-                        if _domain == domain:
-                            continue
-                        for s in _subdomains:
-                            first_segment = s.split(".")[0]
-                            # skip stuff with lots of numbers (e.g. PTRs)
-                            if self.has_excessive_digits(first_segment):
-                                continue
-                            add_mutation(domain_hash, first_segment)
-                            for word in self.helpers.extract_words(
-                                first_segment, word_regexes=self.helpers.word_cloud.dns_mutator.extract_word_regexes
-                            ):
-                                add_mutation(domain_hash, word)
-
-                    num_massdns_mutations = len(mutations) - num_base_mutations
-                    self.debug(f"Mutations from previous subdomains for {domain}: {num_massdns_mutations:,}")
-
-                    # numbers + devops mutations
-                    for mutation in self.helpers.word_cloud.mutations(
-                        subdomains, cloud=False, numbers=3, number_padding=1
-                    ):
-                        for delimiter in ("", ".", "-"):
-                            m = delimiter.join(mutation).lower()
-                            add_mutation(domain_hash, m)
-
-                    num_word_cloud_mutations = len(mutations) - num_massdns_mutations
-                    self.debug(f"Mutations added by word cloud for {domain}: {num_word_cloud_mutations:,}")
-
-                    # special dns mutator
-                    self.debug(
-                        f"DNS Mutator size: {len(self.helpers.word_cloud.dns_mutator):,} (limited to {self.max_mutations:,})"
-                    )
-                    for subdomain in self.helpers.word_cloud.dns_mutator.mutations(
-                        subdomains, max_mutations=self.max_mutations
-                    ):
-                        add_mutation(domain_hash, subdomain)
-
-                    num_mutations = len(mutations) - num_word_cloud_mutations
-                    self.debug(f"Mutations added by DNS Mutator: {num_mutations:,}")
-
-                    if mutations:
-                        self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(found)})")
-                        results = list(await self.massdns(query, mutations))
-                        if results:
-                            await self.resolve_and_emit_queue.put((results, None, [f"mutation-{self.mutation_run}"]))
-                            found_mutations = True
-                            continue
-                    break
-        except AssertionError as e:
-            self.warning(e)
-
-        if found_mutations:
-            self.mutation_run += 1
-
-    def add_found(self, host):
-        if not isinstance(host, str):
-            host = host.data
-        if self.helpers.is_subdomain(host):
-            subdomain, domain = host.split(".", 1)
-            is_ptr = self.helpers.is_ptr(subdomain)
-            in_scope = self.scan.in_scope(domain)
-            if in_scope and not is_ptr:
-                try:
-                    self.found[domain].add(subdomain)
-                except KeyError:
-                    self.found[domain] = set((subdomain,))
-
-    async def gen_subdomains(self, prefixes, domain):
-        for p in prefixes:
-            d = f"{p}.{domain}"
-            yield d
-
-    def gen_random_subdomains(self, n=50):
-        delimiters = (".", "-")
-        lengths = list(range(3, 8))
-        for i in range(0, max(0, n - 5)):
-            d = delimiters[i % len(delimiters)]
-            l = lengths[i % len(lengths)]
-            segments = list(random.choice(self.devops_mutations) for _ in range(l))
-            segments.append(self.helpers.rand_string(length=8, digits=False))
-            subdomain = d.join(segments)
-            yield subdomain
-        for _ in range(5):
-            yield self.helpers.rand_string(length=8, digits=False)
-
-    def has_excessive_digits(self, d):
-        """
-        Identifies dns names with excessive numbers, e.g.:
-            - w1-2-3.evilcorp.com
-            - ptr1234.evilcorp.com
-        """
-        digits = self.digit_regex.findall(d)
-        excessive_digits = len(digits) > 2
-        long_digits = any(len(d) > 3 for d in digits)
-        if excessive_digits or long_digits:
-            return True
-        return False
diff --git a/bbot/modules/output/human.py b/bbot/modules/output/human.py
deleted file mode 100644
index 37b812214..000000000
--- a/bbot/modules/output/human.py
+++ /dev/null
@@ -1,49 +0,0 @@
-from contextlib import suppress
-
-from bbot.core.helpers.logger import log_to_stderr
-from bbot.modules.output.base import BaseOutputModule
-
-
-class Human(BaseOutputModule):
-    watched_events = ["*"]
-    meta = {"description": "Output to text", "created_date": "2022-04-07", "author": "@TheTechromancer"}
-    options = {"output_file": "", "console": True}
-    options_desc = {"output_file": "Output to file", "console": "Output to console"}
-    vuln_severity_map = {"LOW": "HUGEWARNING", "MEDIUM": "HUGEWARNING", "HIGH": "CRITICAL", "CRITICAL": "CRITICAL"}
-    accept_dupes = False
-
-    output_filename = "output.txt"
-
-    async def setup(self):
-        self._prep_output_dir(self.output_filename)
-        return True
-
-    async def handle_event(self, event):
-        event_type = f"[{event.type}]"
-        event_tags = ""
-        if getattr(event, "tags", []):
-            event_tags = f'\t({", ".join(sorted(getattr(event, "tags", [])))})'
-        event_str = f"{event_type:<20}\t{event.data_human}\t{event.module_sequence}{event_tags}"
-        # log vulnerabilities in vivid colors
-        if event.type == "VULNERABILITY":
-            severity = event.data.get("severity", "INFO")
-            if severity in self.vuln_severity_map:
-                loglevel = self.vuln_severity_map[severity]
-                log_to_stderr(event_str, level=loglevel, logname=False)
-        elif event.type == "FINDING":
-            log_to_stderr(event_str, level="HUGEINFO", logname=False)
-
-        if self.file is not None:
-            self.file.write(event_str + "\n")
-            self.file.flush()
-        if self.config.get("console", True):
-            self.stdout(event_str)
-
-    async def cleanup(self):
-        if getattr(self, "_file", None) is not None:
-            with suppress(Exception):
-                self.file.close()
-
-    async def report(self):
-        if getattr(self, "_file", None) is not None:
-            self.info(f"Saved TXT output to {self.output_file}")

From b82ebc23b0a27499d59648e51d61f3751901ab4b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 22 May 2024 17:24:57 -0400
Subject: [PATCH 173/220] fix trufflehog test inconsistencies

---
 bbot/modules/internal/speculate.py            |  2 +-
 bbot/test/test_step_2/module_tests/base.py    |  2 -
 .../module_tests/test_module_dnscommonsrv.py  | 34 +++++++++-
 .../module_tests/test_module_trufflehog.py    | 64 +++++++++++++++----
 4 files changed, 84 insertions(+), 18 deletions(-)

diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 1d09cb46e..dc4dc0371 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -85,7 +85,7 @@ async def handle_event(self, event):
         if event.type == "DNS_NAME":
             parent = self.helpers.parent_domain(event.data)
             if parent != event.data:
-                await self.emit_event(parent, "DNS_NAME", source=event, internal=True)
+                await self.emit_event(parent, "DNS_NAME", source=event)
 
         # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
         event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 8dcabe996..e8628bcf2 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -126,8 +126,6 @@ async def module_test(
 
     @pytest.mark.asyncio
     async def test_module_run(self, module_test):
-        for e in module_test.events:
-            self.log.critical(f"{e} <-- {e.source}")
         self.check(module_test, module_test.events)
         module_test.log.info(f"Finished {self.name} module test")
         current_task = asyncio.current_task()
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
index 8d54f4e3a..2fb67a03e 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
@@ -2,6 +2,9 @@
 
 
 class TestDNSCommonSRV(ModuleTestBase):
+    targets = ["test.api.blacklanternsecurity.com"]
+    whitelist = ["blacklanternsecurity.com"]
+    modules_overrides = ["dnscommonsrv", "speculate"]
     config_overrides = {"dns_resolution": True}
 
     async def setup_after_prep(self, module_test):
@@ -13,6 +16,10 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
                 _input = [l async for l in kwargs["input"]]
                 if "_ldap._tcp.gc._msdcs.blacklanternsecurity.com" in _input:
                     yield """{"name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","type":"SRV","class":"IN","status":"NOERROR","rx_ts":1713974911725326170,"data":{"answers":[{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."},{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."}]},"flags":["rd","ra"],"resolver":"195.226.187.130:53","proto":"UDP"}"""
+                if "_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com" in _input:
+                    yield """{"name":"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com.","type":"SRV","class":"IN","status":"NOERROR","rx_ts":1713974911725326170,"data":{"answers":[{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."},{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."}]},"flags":["rd","ra"],"resolver":"195.226.187.130:53","proto":"UDP"}"""
+                if "_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com" in _input:
+                    yield """{"name":"_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com.","type":"SRV","class":"IN","status":"NOERROR","rx_ts":1713974911725326170,"data":{"answers":[{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."},{"ttl":86400,"type":"SRV","class":"IN","name":"_ldap._tcp.gc._msdcs.blacklanternsecurity.com.","data":"10 10 1720 asdf.blacklanternsecurity.com."}]},"flags":["rd","ra"],"resolver":"195.226.187.130:53","proto":"UDP"}"""
             else:
                 async for _ in old_run_live(*command, check=False, text=True, **kwargs):
                     yield _
@@ -22,13 +29,21 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
         await module_test.mock_dns(
             {
                 "blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "api.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
+                "test.api.blacklanternsecurity.com": {"A": ["1.2.3.4"]},
                 "_ldap._tcp.gc._msdcs.blacklanternsecurity.com": {"SRV": ["0 100 3268 asdf.blacklanternsecurity.com"]},
+                "_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com": {
+                    "SRV": ["0 100 3268 asdf.blacklanternsecurity.com"]
+                },
+                "_ldap._tcp.gc._msdcs.test.api.blacklanternsecurity.com": {
+                    "SRV": ["0 100 3268 asdf.blacklanternsecurity.com"]
+                },
                 "asdf.blacklanternsecurity.com": {"A": ["1.2.3.5"]},
             }
         )
 
     def check(self, module_test, events):
-        assert len(events) == 4
+        assert len(events) == 11
         assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"])
         assert 1 == len(
             [
@@ -44,7 +59,20 @@ def check(self, module_test, events):
                 e
                 for e in events
                 if e.type == "DNS_NAME"
-                and e.data == "asdf.blacklanternsecurity.com"
-                and str(e.module) != "dnscommonsrv"
+                and e.data == "_ldap._tcp.gc._msdcs.api.blacklanternsecurity.com"
+                and str(e.module) == "dnscommonsrv"
             ]
         ), "Failed to detect subdomain 2"
+        assert 2 == len(
+            [e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"]
+        ), "Failed to detect subdomain 3"
+        assert 1 == len(
+            [e for e in events if e.type == "DNS_NAME" and e.data == "api.blacklanternsecurity.com"]
+        ), "Failed to detect subdomain 4"
+        assert 1 == len(
+            [e for e in events if e.type == "DNS_NAME" and e.data == "test.api.blacklanternsecurity.com"]
+        ), "Failed to detect subdomain 5"
+        assert 1 == len(
+            [e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]
+        ), "Failed to detect main domain"
+        assert 2 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and str(e.module) == "speculate"])
diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
index 4d7122276..f874110a3 100644
--- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
+++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
@@ -860,12 +860,32 @@ def check(self, module_test, events):
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        github_workflow_event = [e for e in vuln_events if "bbot" in e.data["description"]][0].source
-        file = Path(github_workflow_event.data["path"])
-        assert file.is_file(), "Destination file does not exist"
-        docker_source_event = [e for e in vuln_events if e.data["host"] == "hub.docker.com"][0].source
-        file = Path(docker_source_event.data["path"])
-        assert file.is_file(), "Destination image does not exist"
+        filesystem_events = [e.source for e in vuln_events if "bbot" in e.data["description"]]
+        assert len(filesystem_events) == 3
+        assert all([e.type == "FILESYSTEM" for e in filesystem_events])
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith("/git_repos/test_keys") and Path(e.data["path"]).is_dir()
+            ]
+        ), "Test keys repo dir does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith("/workflow_logs/blacklanternsecurity/bbot/test.txt")
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Workflow log file does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith("/docker_images/blacklanternsecurity_helloworld_latest.tar")
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Docker image file does not exist"
 
 
 class TestTrufflehog_NonVerified(TestTrufflehog):
@@ -887,9 +907,29 @@ def check(self, module_test, events):
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        github_workflow_event = [e for e in finding_events if "bbot" in e.data["description"]][0].source
-        file = Path(github_workflow_event.data["path"])
-        assert file.is_file(), "Destination file does not exist"
-        docker_source_event = [e for e in finding_events if e.data["host"] == "hub.docker.com"][0].source
-        file = Path(docker_source_event.data["path"])
-        assert file.is_file(), "Destination image does not exist"
+        filesystem_events = [e.source for e in finding_events if "bbot" in e.data["description"]]
+        assert len(filesystem_events) == 3
+        assert all([e.type == "FILESYSTEM" for e in filesystem_events])
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith("/git_repos/test_keys") and Path(e.data["path"]).is_dir()
+            ]
+        ), "Test keys repo dir does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith("/workflow_logs/blacklanternsecurity/bbot/test.txt")
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Workflow log file does not exist"
+        assert 1 == len(
+            [
+                e
+                for e in filesystem_events
+                if e.data["path"].endswith("/docker_images/blacklanternsecurity_helloworld_latest.tar")
+                and Path(e.data["path"]).is_file()
+            ]
+        ), "Docker image file does not exist"

From d82353f56fba5260dde33b0096e3e9e99e4b0a7a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 22 May 2024 17:41:58 -0400
Subject: [PATCH 174/220] fix module basic tests

---
 bbot/test/test_step_1/test_modules_basic.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 54ab1f3d9..c1a3d0531 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -375,10 +375,11 @@ async def handle_event(self, event):
     scan.modules["dummy"] = dummy(scan)
     events = [e async for e in scan.async_start()]
 
-    assert len(events) == 7
+    assert len(events) == 8
     assert 1 == len([e for e in events if e.type == "SCAN"])
-    assert 2 == len([e for e in events if e.type == "DNS_NAME"])
-    assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"])
+    assert 3 == len([e for e in events if e.type == "DNS_NAME"])
+    # one from target and one from speculate
+    assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com"])
     # the reason we don't have a DNS_NAME for www.evilcorp.com is because FINDING.quick_emit = True
     assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.com"])
@@ -388,7 +389,7 @@ async def handle_event(self, event):
 
     assert scan.stats.events_emitted_by_type == {
         "SCAN": 1,
-        "DNS_NAME": 2,
+        "DNS_NAME": 3,
         "URL": 1,
         "URL_UNVERIFIED": 1,
         "FINDING": 1,
@@ -420,17 +421,17 @@ async def handle_event(self, event):
     assert python_stats.produced == {}
     assert python_stats.produced_total == 0
     assert python_stats.consumed == {
-        "DNS_NAME": 2,
+        "DNS_NAME": 3,
         "FINDING": 1,
         "ORG_STUB": 1,
         "SCAN": 1,
         "URL": 1,
         "URL_UNVERIFIED": 1,
     }
-    assert python_stats.consumed_total == 7
+    assert python_stats.consumed_total == 8
 
     speculate_stats = scan.stats.module_stats["speculate"]
-    assert speculate_stats.produced == {"URL_UNVERIFIED": 1, "ORG_STUB": 1}
-    assert speculate_stats.produced_total == 2
+    assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1}
+    assert speculate_stats.produced_total == 3
     assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 2, "URL_UNVERIFIED": 1}
     assert speculate_stats.consumed_total == 4

From a91c7565c61979f016fe8281557a4762a2b568db Mon Sep 17 00:00:00 2001
From: Colin Stubbs <3059577+colin-stubbs@users.noreply.github.com>
Date: Thu, 23 May 2024 09:34:12 +1000
Subject: [PATCH 175/220] add IPv4 pattern, improve IPv6 pattern

Assign IP patterns to IP event types, add testing process for IP related regex
---
 bbot/core/helpers/regexes.py          |  28 ++++++-
 bbot/test/test_step_1/test_regexes.py | 110 ++++++++++++++++++++++++++
 2 files changed, 137 insertions(+), 1 deletion(-)

diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py
index 6e80801a6..df44fc143 100644
--- a/bbot/core/helpers/regexes.py
+++ b/bbot/core/helpers/regexes.py
@@ -20,9 +20,24 @@
 word_num_regex = re.compile(r"[^\W_]+")
 num_regex = re.compile(r"\d+")
 
-_ipv6_regex = r"[A-F0-9:]*:[A-F0-9:]*:[A-F0-9:]*"
+_ipv4_regex = r"([0-9]{1,3}\.){3,3}[0-9]{1,3}"
+ipv4_regex = re.compile(_ipv4_regex, re.I)
+
+# IPv6 is complicated, so we have accomodate alternative patterns,
+# :(:[A-F0-9]{1,4}){1,7} == ::1, ::ffff:1
+# ([A-F0-9]{1,4}:){1,7}: == 2001::, 2001:db8::, 2001:db8:0:1:2:3::
+# ([A-F0-9]{1,4}:){1,6}:([A-F0-9]{1,4}) == 2001::1, 2001:db8::1, 2001:db8:0:1:2:3::1
+# ([A-F0-9]{1,4}:){7,7}([A-F0-9]{1,4}) == 1:1:1:1:1:1:1:1, ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff
+
+_ipv6_regex = r"(:(:[A-F0-9]{1,4}){1,7}|([A-F0-9]{1,4}:){1,7}:|([A-F0-9]{1,4}:){1,6}:([A-F0-9]{1,4})|([A-F0-9]{1,4}:){7,7}([A-F0-9]{1,4}))"
 ipv6_regex = re.compile(_ipv6_regex, re.I)
 
+_ip_range_regexes = (
+    _ipv4_regex + r"\/[0-9]{1,2}",
+    _ipv6_regex + r"\/[0-9]{1,3}",
+)
+ip_range_regexes = list(re.compile(r, re.I) for r in _ip_range_regexes)
+
 # dns names with periods
 _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
 dns_name_regex = re.compile(_dns_name_regex, re.I)
@@ -73,6 +88,17 @@
                 "EMAIL_ADDRESS",
                 (r"^" + _email_regex + r"$",),
             ),
+            (
+                "IP_ADDRESS",
+                (
+                    r"^" + _ipv4_regex + r"$",
+                    r"^" + _ipv6_regex + r"$",
+                ),
+            ),
+            (
+                "IP_RANGE",
+                tuple(r"^" + r + r"$" for r in _ip_range_regexes),
+            ),
             (
                 "OPEN_TCP_PORT",
                 tuple(r"^" + r + r"$" for r in _open_port_regexes),
diff --git a/bbot/test/test_step_1/test_regexes.py b/bbot/test/test_step_1/test_regexes.py
index 7807e6c79..303d2e94a 100644
--- a/bbot/test/test_step_1/test_regexes.py
+++ b/bbot/test/test_step_1/test_regexes.py
@@ -6,6 +6,116 @@
 from bbot.core.errors import ValidationError
 
 
+def test_ip_regexes():
+    bad_ip = [
+        "203.0..113.0",  # double dot typeo
+        ".0.113.0",  # Partial match
+        "203.0.113.",  # Partial match
+        "203.0.113.0:80",  # correctly formatted with :port appended
+        "255.255.255.256",  # octet greater than 255
+        ":2001:db8::",  # leading : included, probably from : separator usage
+        "2001:db8:::80",  # incorrectly formatted with :port appended
+        "[2001:db8::]:80",  # correctly formatted with :port appended
+        "2001:db8:g::",  # includes non-hex character,
+        "9e:3e:53:29:43:64",  # MAC address, poor regex patterning will often detect these.
+    ]
+
+    good_ip = [
+        "127.0.0.1",
+        "203.0.113.0",
+        "203.0.113.0/24",
+        "0.0.0.0",
+        "255.255.255.255",
+        "::1",
+        "2001:db8::",
+        "2001:db8::1",
+        "2001:db8::1/128",
+        "1:1:1:1:1:1:1:1",
+        "1::1",
+        "ffff::ffff",
+        "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
+    ]
+
+    ip_address_regexes = regexes.event_type_regexes["IP_ADDRESS"]
+
+    for ip in bad_ip:
+        for r in ip_address_regexes:
+            assert not r.match(ip), f"BAD IP ADDRESS: {ip} matched regex: {r}"
+
+        try:
+            event_type, _ = get_event_type(ip)
+            if event_type == "OPEN_TCP_PORT":
+                assert ip == "203.0.113.0:80"
+                continue
+            if event_type == "OPEN_TCP_PORT":
+                assert ip == "[2001:db8::]:80"
+                continue
+            if event_type == "DNS_NAME":
+                assert ip == "203.0.113"
+                continue
+            pytest.fail(f"BAD IP ADDRESS: {ip} matched returned event type: {event_type}")
+        except ValidationError:
+            continue
+        except Exception as e:
+            pytest.fail(f"BAD IP ADDRESS: {ip} raised unknown error: {e}")
+
+    for ip in good_ip:
+        matches = list(r.match(ip) for r in ip_address_regexes)
+        assert any(matches), f"Good IP ADDRESS {ip} did not match regexes"
+        event_type, _ = get_event_type(ip)
+        if not event_type == "IP_ADDRESS":
+            assert (
+                ip == "203.0.113.0/24" and event_type == "IP_RANGE"
+            ), f"Event type for IP_ADDRESS {ip} was not properly detected"
+
+
+def test_ip_range_regexes():
+    bad_ip_ranges = [
+        "203.0.113.0",
+        "203.0.113.0/",
+        "203.0.113.0/a",
+        "2001:db8::/",
+        "2001:db8::/a",
+        "evilcorp.com",
+        "[2001:db8::]:80",
+    ]
+
+    good_ip_ranges = [
+        "203.0.113.0/8",
+        "203.0.113.255/32",
+        "2001:db8::/128",
+        "2001:db8::/4",
+    ]
+
+    ip_range_regexes = regexes.event_type_regexes["IP_RANGE"]
+
+    for bad_ip_range in bad_ip_ranges:
+        for r in ip_range_regexes:
+            assert not r.match(bad_ip_range), f"BAD IP_RANGE: {bad_ip_range} matched regex: {r}"
+
+        event_type = ""
+        try:
+            event_type, _ = get_event_type(bad_ip_range)
+            if event_type == "DNS_NAME":
+                assert bad_ip_range == "evilcorp.com"
+                continue
+            if event_type == "IP_ADDRESS":
+                assert bad_ip_range == "203.0.113.0"
+                continue
+            if event_type == "OPEN_TCP_PORT":
+                assert bad_ip_range == "[2001:db8::]:80"
+                continue
+            pytest.fail(f"BAD IP_RANGE: {bad_ip_range} matched returned event type: {event_type}")
+        except ValidationError:
+            continue
+        except Exception as e:
+            pytest.fail(f"BAD IP_RANGE: {bad_ip_range} raised unknown error: {e}: {traceback.format_exc()}")
+
+    for good_ip_range in good_ip_ranges:
+        matches = list(r.match(good_ip_range) for r in ip_range_regexes)
+        assert any(matches), f"Good IP_RANGE {good_ip_range} did not match regexes"
+
+
 def test_dns_name_regexes():
     bad_dns = [
         "-evilcorp.com",  # DNS names cannot begin with a dash

From bacf51083245ca5a042251d03e27d2b5d1857df8 Mon Sep 17 00:00:00 2001
From: Colin Stubbs <3059577+colin-stubbs@users.noreply.github.com>
Date: Thu, 23 May 2024 11:42:35 +1000
Subject: [PATCH 176/220] improve IPv4 regex, adjust testing based on testing

---
 bbot/core/helpers/regexes.py          |  2 +-
 bbot/test/test_step_1/test_regexes.py | 49 ++++++++++++++++++++-------
 2 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py
index df44fc143..890cedf40 100644
--- a/bbot/core/helpers/regexes.py
+++ b/bbot/core/helpers/regexes.py
@@ -20,7 +20,7 @@
 word_num_regex = re.compile(r"[^\W_]+")
 num_regex = re.compile(r"\d+")
 
-_ipv4_regex = r"([0-9]{1,3}\.){3,3}[0-9]{1,3}"
+_ipv4_regex = r"(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}"
 ipv4_regex = re.compile(_ipv4_regex, re.I)
 
 # IPv6 is complicated, so we have accomodate alternative patterns,
diff --git a/bbot/test/test_step_1/test_regexes.py b/bbot/test/test_step_1/test_regexes.py
index 303d2e94a..15f10d4da 100644
--- a/bbot/test/test_step_1/test_regexes.py
+++ b/bbot/test/test_step_1/test_regexes.py
@@ -6,6 +6,10 @@
 from bbot.core.errors import ValidationError
 
 
+# NOTE: :2001:db8:: will currently cause an exception...
+# e.g. raised unknown error: split_port() failed to parse netloc ":2001:db8::"
+
+
 def test_ip_regexes():
     bad_ip = [
         "203.0..113.0",  # double dot typeo
@@ -13,18 +17,26 @@ def test_ip_regexes():
         "203.0.113.",  # Partial match
         "203.0.113.0:80",  # correctly formatted with :port appended
         "255.255.255.256",  # octet greater than 255
-        ":2001:db8::",  # leading : included, probably from : separator usage
+        "256.255.255.255",  # octet greater than 255
         "2001:db8:::80",  # incorrectly formatted with :port appended
         "[2001:db8::]:80",  # correctly formatted with :port appended
         "2001:db8:g::",  # includes non-hex character,
+        "2001.db8.80",  # weird dot separated thing that might actually resolve as a DNS_NAME
         "9e:3e:53:29:43:64",  # MAC address, poor regex patterning will often detect these.
     ]
 
     good_ip = [
+        "0.0.0.0",
+        "10.0.0.0",
+        "10.255.255.255",
+        "127.0.0.0",
         "127.0.0.1",
+        "172.16.0.0",
+        "172.31.255.255",
+        "192.168.0.0",
+        "192.168.255.255",
         "203.0.113.0",
         "203.0.113.0/24",
-        "0.0.0.0",
         "255.255.255.255",
         "::1",
         "2001:db8::",
@@ -45,13 +57,20 @@ def test_ip_regexes():
         try:
             event_type, _ = get_event_type(ip)
             if event_type == "OPEN_TCP_PORT":
-                assert ip == "203.0.113.0:80"
-                continue
-            if event_type == "OPEN_TCP_PORT":
-                assert ip == "[2001:db8::]:80"
+                if ip.startswith("["):
+                    assert ip == "[2001:db8::]:80"
+                else:
+                    assert ip == "203.0.113.0:80"
                 continue
             if event_type == "DNS_NAME":
-                assert ip == "203.0.113"
+                if ip.startswith("2001"):
+                    assert ip == "2001.db8.80"
+                elif ip.startswith("255"):
+                    assert ip == "255.255.255.256"
+                elif ip.startswith("256"):
+                    assert ip == "256.255.255.255"
+                else:
+                    assert ip == "203.0.113."
                 continue
             pytest.fail(f"BAD IP ADDRESS: {ip} matched returned event type: {event_type}")
         except ValidationError:
@@ -60,13 +79,19 @@ def test_ip_regexes():
             pytest.fail(f"BAD IP ADDRESS: {ip} raised unknown error: {e}")
 
     for ip in good_ip:
-        matches = list(r.match(ip) for r in ip_address_regexes)
-        assert any(matches), f"Good IP ADDRESS {ip} did not match regexes"
         event_type, _ = get_event_type(ip)
         if not event_type == "IP_ADDRESS":
-            assert (
-                ip == "203.0.113.0/24" and event_type == "IP_RANGE"
-            ), f"Event type for IP_ADDRESS {ip} was not properly detected"
+            if ip.endswith("/24"):
+                assert (
+                    ip == "203.0.113.0/24" and event_type == "IP_RANGE"
+                ), f"Event type for IP_ADDRESS {ip} was not properly detected"
+            else:
+                assert (
+                    ip == "2001:db8::1/128" and event_type == "IP_RANGE"
+                ), f"Event type for IP_ADDRESS {ip} was not properly detected"
+        else:
+            matches = list(r.match(ip) for r in ip_address_regexes)
+            assert any(matches), f"Good IP ADDRESS {ip} did not match regexes"
 
 
 def test_ip_range_regexes():

From 0b48ff1e813f30ddb51dfce3a6f24179c7cfdbd9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 22 May 2024 22:58:47 -0400
Subject: [PATCH 177/220] fix tests

---
 bbot/modules/internal/speculate.py                     |  8 +-------
 .../module_tests/test_module_dnscommonsrv.py           | 10 ++++++++--
 .../template_tests/test_template_subdomain_enum.py     |  3 ++-
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index dc4dc0371..28fcc4758 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -82,7 +82,7 @@ async def handle_event(self, event):
                 await self.emit_event(ip, "IP_ADDRESS", source=event, internal=True)
 
         # parent domains
-        if event.type == "DNS_NAME":
+        if event.type.startswith("DNS_NAME"):
             parent = self.helpers.parent_domain(event.data)
             if parent != event.data:
                 await self.emit_event(parent, "DNS_NAME", source=event)
@@ -182,9 +182,3 @@ async def handle_event(self, event):
                 if email_event:
                     email_event.scope_distance = event.scope_distance
                     await self.emit_event(email_event)
-
-    async def filter_event(self, event):
-        # don't accept errored DNS_NAMEs
-        if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")):
-            return False, "there were errors resolving this hostname"
-        return True
diff --git a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
index 2fb67a03e..dbbca6de5 100644
--- a/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py
@@ -39,11 +39,12 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
                     "SRV": ["0 100 3268 asdf.blacklanternsecurity.com"]
                 },
                 "asdf.blacklanternsecurity.com": {"A": ["1.2.3.5"]},
+                "_msdcs.api.blacklanternsecurity.com": {"A": ["1.2.3.5"]},
             }
         )
 
     def check(self, module_test, events):
-        assert len(events) == 11
+        assert len(events) == 15
         assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"])
         assert 1 == len(
             [
@@ -72,7 +73,12 @@ def check(self, module_test, events):
         assert 1 == len(
             [e for e in events if e.type == "DNS_NAME" and e.data == "test.api.blacklanternsecurity.com"]
         ), "Failed to detect subdomain 5"
+        assert 1 == len(
+            [e for e in events if e.type == "DNS_NAME" and e.data == "_msdcs.api.blacklanternsecurity.com"]
+        ), "Failed to detect subdomain 5"
         assert 1 == len(
             [e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]
         ), "Failed to detect main domain"
-        assert 2 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and str(e.module) == "speculate"])
+        assert 8 == len([e for e in events if e.type == "DNS_NAME"])
+        assert 5 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED"])
+        assert 5 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and str(e.module) == "speculate"])
diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
index 026895108..1ef87d3d1 100644
--- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
+++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
@@ -68,8 +68,9 @@ def check(self, module_test, events):
         assert 1 == len([e for e in in_scope_dns_names if e.data == "asdf.www.blacklanternsecurity.com"])
         assert 1 == len([e for e in in_scope_dns_names if e.data == "test.asdf.www.blacklanternsecurity.com"])
         assert 1 == len([e for e in in_scope_dns_names if e.data == "api.test.asdf.www.blacklanternsecurity.com"])
-        assert len(distance_1_dns_names) == 1
+        assert len(distance_1_dns_names) == 2
         assert 1 == len([e for e in distance_1_dns_names if e.data == "evilcorp.com"])
+        assert 1 == len([e for e in distance_1_dns_names if e.data == "blacklanternsecurity.com"])
         assert len(self.queries) == 1
         assert self.queries[0] == "www.blacklanternsecurity.com"
 

From 768886ba6615d62178e67874ff355fb6b7c84c1f Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 23 May 2024 18:47:30 -0400
Subject: [PATCH 178/220] WIP event discovery context

---
 bbot/core/event/base.py              | 10 +++++++---
 bbot/modules/base.py                 |  2 +-
 bbot/modules/deadly/dastardly.py     |  2 ++
 bbot/modules/httpx.py                | 24 ++++++++++++++++++++++--
 bbot/modules/internal/dns.py         |  8 +++++++-
 bbot/scanner/manager.py              |  7 +++++--
 bbot/test/test_step_1/test_events.py |  6 ++++++
 7 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ca1963587..e4a7f3640 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -217,10 +217,10 @@ def __init__(
         # inherit web spider distance from parent
         self.web_spider_distance = getattr(self.parent, "web_spider_distance", 0)
 
-        if (not context) and (not self._dummy):
-            raise ValidationError(f"Must specify event discovery context")
+        if not context:
+            context = getattr(self.module, "default_discovery_context", "")
         self._discovery_context = ""
-        if context is not None:
+        if context:
             self.discovery_context = context
 
     @property
@@ -509,6 +509,8 @@ def get_parents(self, omit=False):
                 parent = e.get_parent()
             else:
                 parent = e.parent
+            if parent is None:
+                break
             if e == parent:
                 break
             parents.append(parent)
@@ -686,6 +688,8 @@ def json(self, mode="json", siem_friendly=False):
         # sequence of modules that led to discovery
         if self.module_sequence:
             j.update({"module_sequence": str(self.module_sequence)})
+        # discovery context
+        j["discovery_context"] = self.full_discovery_context
 
         # normalize non-primitive python objects
         for k, v in list(j.items()):
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 520f555ac..8990ace5b 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -427,7 +427,7 @@ def make_event(self, *args, **kwargs):
             ValidationError: If the event could not be validated and raise_error is True.
         """
         raise_error = kwargs.pop("raise_error", False)
-        context = kwargs.pop("context", None)
+        context = kwargs.get("context", None)
         if context is None:
             kwargs["context"] = self.default_discovery_context
         module = kwargs.pop("module", None)
diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py
index 6be92033f..c6ccc383d 100644
--- a/bbot/modules/deadly/dastardly.py
+++ b/bbot/modules/deadly/dastardly.py
@@ -12,6 +12,8 @@ class dastardly(BaseModule):
     deps_common = ["docker"]
     per_hostport_only = True
 
+    default_discovery_context = "{module} performed a light web scan against {event.parent.data['url']} and discovered {event.data['description']} at {event.data['url']}"
+
     async def setup(self):
         await self.run_process("systemctl", "start", "docker", sudo=True)
         await self.run_process("docker", "pull", "public.ecr.aws/portswigger/dastardly:latest", sudo=True)
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index 897625599..01d373078 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -168,12 +168,32 @@ async def handle_batch(self, *events):
             title = self.helpers.tagify(j.get("title", ""), maxlen=30)
             if title:
                 tags.append(f"http-title-{title}")
-            url_event = self.make_event(url, "URL", parent_event, tags=tags)
+
+            url_context = "{module} visited {event.parent.data} and got status code {event.http_status}"
+            if parent_event.type == "OPEN_TCP_PORT":
+                url_context += " at {event.data}"
+
+            url_event = self.make_event(
+                url,
+                "URL",
+                parent_event,
+                tags=tags,
+                context=url_context,
+            )
             if url_event:
                 if url_event != parent_event:
                     await self.emit_event(url_event)
                 # HTTP response
-                await self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags)
+                content_type = j.get("header", {}).get("content_type", "unspecified")
+                content_length = j.get("content_length", 0)
+                content_length = self.helpers.bytes_to_human(content_length)
+                await self.emit_event(
+                    j,
+                    "HTTP_RESPONSE",
+                    url_event,
+                    tags=url_event.tags,
+                    context=f"HTTP_RESPONSE was {content_length} with content type {content_type}",
+                )
 
         for tempdir in Path(tempfile.gettempdir()).iterdir():
             if tempdir.is_dir() and self.httpx_tempdir_regex.match(tempdir.name):
diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index e1fdb7721..0f852fe63 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -175,7 +175,13 @@ async def handle_event(self, event, kwargs):
             and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE")
             and not ((event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate"))
         ):
-            parent_event = self.scan.make_event(event.host, "DNS_NAME", module=self.host_module, parent=event)
+            parent_event = self.scan.make_event(
+                event.host,
+                "DNS_NAME",
+                module=self.host_module,
+                parent=event,
+                context="{event.parent.type} has host {event.type} {event.host}",
+            )
             # only emit the event if it's not already in the parent chain
             if parent_event is not None and (parent_event.always_emit or parent_event not in event.get_parents()):
                 parent_event.scope_distance = event.scope_distance
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 8de5c1ef2..56459cc67 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -66,8 +66,11 @@ async def handle_event(self, event, kwargs):
             return False, "cannot emit dummy event"
 
         # don't accept events with self as parent
-        if (not event.type == "SCAN") and (event == event.get_parent()):
-            return False, "event's parent is itself"
+        if not event.type == "SCAN":
+            if event == event.get_parent():
+                return False, "event's parent is itself"
+            if not event.discovery_context:
+                self.warning(f"Event {event} has no discovery context")
 
         # don't accept duplicates
         if self.is_incoming_duplicate(event, add=True):
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 8d027c3a6..68d1c4809 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -210,6 +210,12 @@ async def test_events(events, helpers):
         assert tag in affiliate_event2.tags
         assert tag not in affiliate_event3.tags
 
+    # discovery context
+    event = scan.make_event(
+        "127.0.0.1", parent=scan.root_event, context="something discovered {event.type}: {event.data}"
+    )
+    assert event.discovery_context == "something discovered IP_ADDRESS: 127.0.0.1"
+
     # updating an already-created event with make_event()
     # updating tags
     event1 = scan.make_event("127.0.0.1", parent=scan.root_event)

From 1f52dc902f985e1346c1dfb841fcbf1d068990fa Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 24 May 2024 02:15:51 -0400
Subject: [PATCH 179/220] event context for excavate

---
 bbot/core/event/base.py            |  2 +-
 bbot/modules/httpx.py              |  4 +-
 bbot/modules/internal/excavate.py  | 67 ++++++++++++++++++++++++++----
 bbot/modules/internal/speculate.py |  2 +
 4 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index e4a7f3640..ae145db54 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1342,7 +1342,7 @@ def make_event(
         if parent is not None:
             data.parent = parent
         if context is not None:
-            data.context = context
+            data.discovery_context = context
         if internal == True:
             data.internal = True
         if tags:
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index 01d373078..4269ff711 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -184,7 +184,7 @@ async def handle_batch(self, *events):
                 if url_event != parent_event:
                     await self.emit_event(url_event)
                 # HTTP response
-                content_type = j.get("header", {}).get("content_type", "unspecified")
+                content_type = j.get("header", {}).get("content_type", "unspecified").split(";")[0]
                 content_length = j.get("content_length", 0)
                 content_length = self.helpers.bytes_to_human(content_length)
                 await self.emit_event(
@@ -192,7 +192,7 @@ async def handle_batch(self, *events):
                     "HTTP_RESPONSE",
                     url_event,
                     tags=url_event.tags,
-                    context=f"HTTP_RESPONSE was {content_length} with content type {content_type}",
+                    context=f"HTTP_RESPONSE was {content_length} with {content_type} content type",
                 )
 
         for tempdir in Path(tempfile.gettempdir()).iterdir():
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 60d9ad73b..b06513985 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -52,7 +52,13 @@ async def search(self, content, event, **kwargs):
                 await self.report(domain, event, **kwargs)
 
     async def report(self, domain, event, **kwargs):
-        await self.excavate.emit_event(domain, "DNS_NAME", parent=event, tags=["affiliate"])
+        await self.excavate.emit_event(
+            domain,
+            "DNS_NAME",
+            parent=event,
+            tags=["affiliate"],
+            context=f"{{module}}'s CSP extractor found {{event.type}}: {{event.data}} in CSP header",
+        )
 
 
 class HostnameExtractor(BaseExtractor):
@@ -64,7 +70,13 @@ def __init__(self, excavate):
         super().__init__(excavate)
 
     async def report(self, result, name, event, **kwargs):
-        await self.excavate.emit_event(result, "DNS_NAME", parent=event)
+        context = kwargs.get("discovery_context", "")
+        await self.excavate.emit_event(
+            result,
+            "DNS_NAME",
+            parent=event,
+            context=f"{{module}}'s hostname extractor found {{event.type}}: {{event.data}} from {context} using regex derived from target domain",
+        )
 
 
 class URLExtractor(BaseExtractor):
@@ -117,7 +129,9 @@ async def search(self, content, event, **kwargs):
                     url_event.add_tag("spider-danger")
 
                 self.excavate.debug(f"Found URL [{result}] from parsing [{event.data.get('url')}] with regex [{name}]")
-                await self.excavate.emit_event(url_event)
+                await self.excavate.emit_event(
+                    url_event,
+                )
                 if url_in_scope:
                     urls_found += 1
 
@@ -153,6 +167,7 @@ async def _search(self, content, event, **kwargs):
                 yield result, name
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         parsed_uri = None
         try:
             parsed_uri = self.excavate.helpers.urlparse(result)
@@ -175,6 +190,7 @@ async def report(self, result, name, event, **kwargs):
                 "FINDING",
                 parent=event,
                 abort_if=abort_if,
+                context=f"{{module}}'s URL extractor found {{event.type}}: Non-HTTP URI {{event.data}} in {context}",
             )
             protocol_data = {"protocol": parsed_uri.scheme, "host": str(host)}
             if port:
@@ -184,10 +200,16 @@ async def report(self, result, name, event, **kwargs):
                 "PROTOCOL",
                 parent=event,
                 abort_if=abort_if,
+                context=f"{{module}}'s URL extractor found {{event.type}}: {{event.data}} in {result} while searching {context}",
             )
             return
 
-        return self.excavate.make_event(result, "URL_UNVERIFIED", parent=event)
+        return self.excavate.make_event(
+            result,
+            "URL_UNVERIFIED",
+            parent=event,
+            context=f"{{module}}'s URL extractor ({name} regex) found {{event.type}}: {{event.data}} in {context}",
+        )
 
 
 class EmailExtractor(BaseExtractor):
@@ -195,11 +217,17 @@ class EmailExtractor(BaseExtractor):
     tld_blacklist = ["png", "jpg", "jpeg", "bmp", "ico", "gif", "svg", "css", "ttf", "woff", "woff2"]
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         result = result.lower()
         tld = result.split(".")[-1]
         if tld not in self.tld_blacklist:
             self.excavate.debug(f"Found email address [{result}] from parsing [{event.data.get('url')}]")
-            await self.excavate.emit_event(result, "EMAIL_ADDRESS", parent=event)
+            await self.excavate.emit_event(
+                result,
+                "EMAIL_ADDRESS",
+                parent=event,
+                context=f"{{module}}'s email extractor found {{event.type}}: {{event.data}} in {context}",
+            )
 
 
 class ErrorExtractor(BaseExtractor):
@@ -221,12 +249,14 @@ class ErrorExtractor(BaseExtractor):
     }
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         self.excavate.debug(f"Found error message from parsing [{event.data.get('url')}] with regex [{name}]")
         description = f"Error message Detected at Error Type: {name}"
         await self.excavate.emit_event(
             {"host": str(event.host), "url": event.data.get("url", ""), "description": description},
             "FINDING",
             parent=event,
+            context=f"{{module}}'s error message extractor detected {{event.type}}: {name} error message in {context}",
         )
 
 
@@ -234,6 +264,7 @@ class JWTExtractor(BaseExtractor):
     regexes = {"JWT": r"eyJ(?:[\w-]*\.)(?:[\w-]*\.)[\w-]*"}
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         self.excavate.debug(f"Found JWT candidate [{result}]")
         try:
             j.decode(result, options={"verify_signature": False})
@@ -247,6 +278,7 @@ async def report(self, result, name, event, **kwargs):
                 "FINDING",
                 event,
                 tags=tags,
+                context=f"{{module}}'s JWT extractor found {{event.type}}: {name} in {context}",
             )
 
         except j.exceptions.DecodeError:
@@ -264,9 +296,13 @@ class SerializationExtractor(BaseExtractor):
     }
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         description = f"{name} serialized object found: [{self.excavate.helpers.truncate_string(result,2000)}]"
         await self.excavate.emit_event(
-            {"host": str(event.host), "url": event.data.get("url"), "description": description}, "FINDING", event
+            {"host": str(event.host), "url": event.data.get("url"), "description": description},
+            "FINDING",
+            event,
+            context=f"{{module}}'s serialization extractor found {{event.type}}: {name} serialized object in {context}",
         )
 
 
@@ -277,9 +313,13 @@ class FunctionalityExtractor(BaseExtractor):
     }
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         description = f"{name} found"
         await self.excavate.emit_event(
-            {"host": str(event.host), "url": event.data.get("url"), "description": description}, "FINDING", event
+            {"host": str(event.host), "url": event.data.get("url"), "description": description},
+            "FINDING",
+            event,
+            context=f"{{module}}'s functionality extractor discovered {{event.type}}: {name} in {context}",
         )
 
 
@@ -319,6 +359,7 @@ class JavascriptExtractor(BaseExtractor):
     }
 
     async def report(self, result, name, event, **kwargs):
+        context = kwargs.get("discovery_context", "")
         # ensure that basic auth matches aren't false positives
         if name == "authorization_basic":
             try:
@@ -331,7 +372,10 @@ async def report(self, result, name, event, **kwargs):
         self.excavate.debug(f"Found Possible Secret in Javascript [{result}]")
         description = f"Possible secret in JS [{result}] Signature [{name}]"
         await self.excavate.emit_event(
-            {"host": str(event.host), "url": event.data.get("url", ""), "description": description}, "FINDING", event
+            {"host": str(event.host), "url": event.data.get("url", ""), "description": description},
+            "FINDING",
+            event,
+            context=f"{{module}}'s Javascript extractor discovered {{event.type}}: {description} in {context}",
         )
 
 
@@ -383,7 +427,10 @@ async def handle_event(self, event):
                             # inherit web spider distance from parent (don't increment)
                             parent_web_spider_distance = getattr(event, "web_spider_distance", 0)
                             url_event.web_spider_distance = parent_web_spider_distance
-                            await self.emit_event(url_event)
+                            await self.emit_event(
+                                url_event,
+                                context="{module} extracted {event.type}: {event.data} from redirect destination (Location header)",
+                            )
                     else:
                         self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}")
 
@@ -403,6 +450,7 @@ async def handle_event(self, event):
                 ],
                 event,
                 consider_spider_danger=True,
+                discovery_context="HTTP response body",
             )
 
             headers = await self.helpers.re.recursive_decode(event.data.get("raw_header", ""))
@@ -411,6 +459,7 @@ async def handle_event(self, event):
                 [self.hostname, self.url, self.email, self.error_extractor, self.jwt, self.serialization, self.csp],
                 event,
                 consider_spider_danger=False,
+                discovery_context="HTTP response headers",
             )
 
         else:
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 601b9ec81..282b94cdb 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -36,6 +36,8 @@ class speculate(BaseInternalModule):
     scope_distance_modifier = 1
     _priority = 4
 
+    default_discovery_context = "{module} speculated {event.type}: {event.data}"
+
     async def setup(self):
         scan_modules = [m for m in self.scan.modules.values() if m._type == "scan"]
         self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in scan_modules])

From 03dd785a020b5ccb6f4fc00e07c688a128c89480 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 24 May 2024 10:51:50 -0400
Subject: [PATCH 180/220] merge bbot 2.0

---
 .../test_step_2/module_tests/test_module_trufflehog.py   | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
index 40b514e58..8e7e70c35 100644
--- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
+++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
@@ -907,14 +907,6 @@ def check(self, module_test, events):
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-<<<<<<< HEAD
-        github_workflow_event = [e for e in finding_events if "bbot" in e.data["description"]][0].parent
-        file = Path(github_workflow_event.data["path"])
-        assert file.is_file(), "Destination file does not exist"
-        docker_parent_event = [e for e in finding_events if e.data["host"] == "hub.docker.com"][0].parent
-        file = Path(docker_parent_event.data["path"])
-        assert file.is_file(), "Destination image does not exist"
-=======
         filesystem_events = [e.source for e in finding_events if "bbot" in e.data["description"]]
         assert len(filesystem_events) == 3
         assert all([e.type == "FILESYSTEM" for e in filesystem_events])
@@ -941,4 +933,3 @@ def check(self, module_test, events):
                 and Path(e.data["path"]).is_file()
             ]
         ), "Docker image file does not exist"
->>>>>>> bbot-2.0

From 57fec0342a322ea30a2667d89d023cec72ef4ad1 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 24 May 2024 10:54:19 -0400
Subject: [PATCH 181/220] fix trufflehog tests

---
 bbot/test/test_step_2/module_tests/test_module_trufflehog.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
index 8e7e70c35..9cd5be601 100644
--- a/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
+++ b/bbot/test/test_step_2/module_tests/test_module_trufflehog.py
@@ -860,7 +860,7 @@ def check(self, module_test, events):
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        filesystem_events = [e.source for e in vuln_events if "bbot" in e.data["description"]]
+        filesystem_events = [e.parent for e in vuln_events if "bbot" in e.data["description"]]
         assert len(filesystem_events) == 3
         assert all([e.type == "FILESYSTEM" for e in filesystem_events])
         assert 1 == len(
@@ -907,7 +907,7 @@ def check(self, module_test, events):
         with open(folder / "keys.txt") as f:
             content = f.read()
             assert content == self.file_content, "File content doesn't match"
-        filesystem_events = [e.source for e in finding_events if "bbot" in e.data["description"]]
+        filesystem_events = [e.parent for e in finding_events if "bbot" in e.data["description"]]
         assert len(filesystem_events) == 3
         assert all([e.type == "FILESYSTEM" for e in filesystem_events])
         assert 1 == len(

From 52416c706faf9fc1df92a3fd334ad5e72e479bd6 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 24 May 2024 13:03:58 -0400
Subject: [PATCH 182/220] fix event tests

---
 bbot/core/event/base.py              |  5 ++--
 bbot/modules/base.py                 |  3 --
 bbot/test/test_step_1/test_events.py | 43 ++++++++++++++--------------
 3 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ae145db54..e3b22031b 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -339,7 +339,7 @@ def discovery_context(self, context):
             self._discovery_context = context
 
     @property
-    def full_discovery_context(self):
+    def discovery_path(self):
         """
         This event's full discovery context, including those of all its parents
         """
@@ -689,7 +689,8 @@ def json(self, mode="json", siem_friendly=False):
         if self.module_sequence:
             j.update({"module_sequence": str(self.module_sequence)})
         # discovery context
-        j["discovery_context"] = self.full_discovery_context
+        j["discovery_context"] = self.discovery_context
+        j["discovery_path"] = self.discovery_path
 
         # normalize non-primitive python objects
         for k, v in list(j.items()):
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 8990ace5b..c0b0b399b 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -427,9 +427,6 @@ def make_event(self, *args, **kwargs):
             ValidationError: If the event could not be validated and raise_error is True.
         """
         raise_error = kwargs.pop("raise_error", False)
-        context = kwargs.get("context", None)
-        if context is None:
-            kwargs["context"] = self.default_discovery_context
         module = kwargs.pop("module", None)
         if module is None:
             kwargs["module"] = self
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 68d1c4809..cd0165180 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -588,7 +588,7 @@ async def handle_event(self, event):
             if e.type == "DNS_NAME"
             and e.data == "evilcorp.com"
             and e.discovery_context == f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"
-            and e.full_discovery_context == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
+            and e.discovery_path == [f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com"]
         ]
     )
     assert 1 == len(
@@ -598,7 +598,7 @@ async def handle_event(self, event):
             if e.type == "DNS_NAME"
             and e.data == "one.evilcorp.com"
             and e.discovery_context == "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com"
-            and e.full_discovery_context
+            and e.discovery_path
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -613,7 +613,7 @@ async def handle_event(self, event):
             and e.data == "two.evilcorp.com"
             and e.discovery_context
             == "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com"
-            and e.full_discovery_context
+            and e.discovery_path
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -628,7 +628,7 @@ async def handle_event(self, event):
             if e.type == "DNS_NAME"
             and e.data == "three.evilcorp.com"
             and e.discovery_context == "module_2 asked nicely and was given DNS_NAME three.evilcorp.com"
-            and e.full_discovery_context
+            and e.discovery_path
             == [
                 f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
                 "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
@@ -637,20 +637,21 @@ async def handle_event(self, event):
             ]
         ]
     )
-    assert 1 == len(
-        [
-            e
-            for e in events
-            if e.type == "DNS_NAME"
-            and e.data == "four.evilcorp.com"
-            and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com"
-            and e.full_discovery_context
-            == [
-                f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
-                "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
-                "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com",
-                "module_2 asked nicely and was given DNS_NAME three.evilcorp.com",
-                "module_2 used brute force to obtain DNS_NAME four.evilcorp.com",
-            ]
-        ]
-    )
+    final_path = [
+        f"Scan {scan.name} seeded with DNS_NAME: evilcorp.com",
+        "module_1 invoked forbidden magick to discover DNS_NAME one.evilcorp.com",
+        "module_1 pledged its allegiance to cthulu and was awarded DNS_NAME two.evilcorp.com",
+        "module_2 asked nicely and was given DNS_NAME three.evilcorp.com",
+        "module_2 used brute force to obtain DNS_NAME four.evilcorp.com",
+    ]
+    final_event = [
+        e
+        for e in events
+        if e.type == "DNS_NAME"
+        and e.data == "four.evilcorp.com"
+        and e.discovery_context == "module_2 used brute force to obtain DNS_NAME four.evilcorp.com"
+        and e.discovery_path == final_path
+    ]
+    assert 1 == len(final_event)
+    j = final_event[0].json()
+    assert j["discovery_path"] == final_path

From eedc6cb54bc9860b8d3ac831d3bb02412f4c0109 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 24 May 2024 16:33:26 -0400
Subject: [PATCH 183/220] steady work on event context

---
 bbot/modules/internal/dns.py       |  5 ++++-
 bbot/modules/internal/speculate.py | 28 ++++++++++++++++++++--------
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index 0f852fe63..ccfe03496 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -180,7 +180,7 @@ async def handle_event(self, event, kwargs):
                 "DNS_NAME",
                 module=self.host_module,
                 parent=event,
-                context="{event.parent.type} has host {event.type} {event.host}",
+                context="{event.parent.type} has host {event.type}: {event.host}",
             )
             # only emit the event if it's not already in the parent chain
             if parent_event is not None and (parent_event.always_emit or parent_event not in event.get_parents()):
@@ -199,6 +199,9 @@ async def handle_event(self, event, kwargs):
                     for record in records:
                         try:
                             child_event = self.scan.make_event(record, "DNS_NAME", module=module, parent=parent_event)
+                            child_event.discovery_context = (
+                                f"{rdtype} record for {event.host} contains {child_event.type}: {child_event.host}"
+                            )
                             # if it's a hostname and it's only one hop away, mark it as affiliate
                             if child_event.type == "DNS_NAME" and child_event.scope_distance == 1:
                                 child_event.add_tag("affiliate")
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 8ae06226e..310cee7d3 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -40,7 +40,7 @@ class speculate(BaseInternalModule):
     scope_distance_modifier = 1
     _priority = 4
 
-    default_discovery_context = "{module} speculated {event.type}: {event.data}"
+    default_discovery_context = "speculated {event.type}: {event.data}"
 
     async def setup(self):
         scan_modules = [m for m in self.scan.modules.values() if m._type == "scan"]
@@ -81,13 +81,21 @@ async def handle_event(self, event):
             ips = list(net)
             random.shuffle(ips)
             for ip in ips:
-                await self.emit_event(ip, "IP_ADDRESS", parent=event, internal=True)
+                await self.emit_event(
+                    ip,
+                    "IP_ADDRESS",
+                    parent=event,
+                    internal=True,
+                    context=f"speculate converted range into indivudal IP_ADDRESS: {ip}",
+                )
 
         # parent domains
         if event.type.startswith("DNS_NAME"):
             parent = self.helpers.parent_domain(event.data)
             if parent != event.data:
-                await self.emit_event(parent, "DNS_NAME", parent=event)
+                await self.emit_event(
+                    parent, "DNS_NAME", parent=event, context=f"speculated parent {{event.type}}: {{event.data}}"
+                )
 
         # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
         event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
@@ -103,6 +111,7 @@ async def handle_event(self, event):
                     parent=event,
                     internal=True,
                     quick=(event.type == "URL"),
+                    context=f"speculated {{event.type}} from {event.type}: {{event.data}}",
                 )
 
         # speculate sub-directory URLS from URLS
@@ -114,7 +123,7 @@ async def handle_event(self, event):
                     # inherit web spider distance from parent (don't increment)
                     parent_web_spider_distance = getattr(event, "web_spider_distance", 0)
                     url_event.web_spider_distance = parent_web_spider_distance
-                    await self.emit_event(url_event)
+                    await self.emit_event(url_event, context="speculated web sub-directory {event.type}: {event.data}")
 
         # speculate URL_UNVERIFIED from URL or any event with "url" attribute
         event_is_url = event.type == "URL"
@@ -128,9 +137,11 @@ async def handle_event(self, event):
                 tags = None
                 if self.helpers.is_spider_danger(event.parent, url):
                     tags = ["spider-danger"]
-                await self.emit_event(url, "URL_UNVERIFIED", tags=tags, parent=event)
+                await self.emit_event(
+                    url, "URL_UNVERIFIED", tags=tags, parent=event, context="speculated {event.type}: {event.data}"
+                )
 
-        # from hosts
+        # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
         if speculate_open_ports:
             # don't act on unresolved DNS_NAMEs
             usable_dns = False
@@ -146,6 +157,7 @@ async def handle_event(self, event):
                         parent=event,
                         internal=True,
                         quick=True,
+                        context="speculated {event.type}: {event.data}",
                     )
 
         # ORG_STUB from TLD, SOCIAL, AZURE_TENANT
@@ -174,7 +186,7 @@ async def handle_event(self, event):
                 if stub_event:
                     if event.scope_distance > 0:
                         stub_event.scope_distance = event.scope_distance
-                    await self.emit_event(stub_event)
+                    await self.emit_event(stub_event, context="speculated {event.type}: {event.data}")
 
         # USERNAME --> EMAIL
         if event.type == "USERNAME":
@@ -183,4 +195,4 @@ async def handle_event(self, event):
                 email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=["affiliate"])
                 if email_event:
                     email_event.scope_distance = event.scope_distance
-                    await self.emit_event(email_event)
+                    await self.emit_event(email_event, context="detected {event.type}: {event.data}")

From 002308ca659751b99a7d30d7423d73bf5d547095 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Fri, 24 May 2024 17:40:50 -0400
Subject: [PATCH 184/220] code enum contexts

---
 bbot/core/flags.py                |  1 -
 bbot/modules/code_repository.py   |  7 ++---
 bbot/modules/dockerhub.py         | 22 +++++++++++++---
 bbot/modules/git.py               | 15 +++++------
 bbot/modules/github_codesearch.py |  7 +++--
 bbot/modules/github_org.py        | 21 +++++++++++----
 bbot/modules/gitlab.py            | 18 ++++++++++---
 bbot/modules/postman.py           | 44 ++++++++++++++++++++++++++-----
 bbot/modules/trufflehog.py        | 14 ++++++++--
 9 files changed, 113 insertions(+), 36 deletions(-)

diff --git a/bbot/core/flags.py b/bbot/core/flags.py
index c16451499..f65dbad28 100644
--- a/bbot/core/flags.py
+++ b/bbot/core/flags.py
@@ -15,7 +15,6 @@
     "service-enum": "Identifies protocols running on open ports",
     "slow": "May take a long time to complete",
     "social-enum": "Enumerates social media",
-    "repo-enum": "Enumerates code repositories",
     "subdomain-enum": "Enumerates subdomains",
     "subdomain-hijack": "Detects hijackable subdomains",
     "web-basic": "Basic, non-intrusive web scan functionality",
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
index 5ce407e8d..2407172bc 100644
--- a/bbot/modules/code_repository.py
+++ b/bbot/modules/code_repository.py
@@ -10,7 +10,7 @@ class code_repository(BaseModule):
         "created_date": "2024-05-15",
         "author": "@domwhewell-sage",
     }
-    flags = ["passive", "safe", "repo-enum"]
+    flags = ["passive", "safe", "code-enum"]
 
     # platform name : (regex, case_sensitive)
     code_repositories = {
@@ -42,11 +42,12 @@ async def handle_event(self, event):
                     url = match.group()
                     if not case_sensitive:
                         url = url.lower()
+                    url = f"https://{url}"
                     repo_event = self.make_event(
-                        {"url": f"https://{url}"},
+                        {"url": url},
                         "CODE_REPOSITORY",
                         tags=platform,
                         parent=event,
                     )
                     repo_event.scope_distance = event.scope_distance
-                    await self.emit_event(repo_event)
+                    await self.emit_event(repo_event, context=f"detected {{event.type}}: {url}")
diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py
index e44422626..41bc027de 100644
--- a/bbot/modules/dockerhub.py
+++ b/bbot/modules/dockerhub.py
@@ -40,7 +40,10 @@ async def handle_org_stub(self, event):
                 site_url = f"{self.site_url}/u/{p}"
                 # emit social event
                 await self.emit_event(
-                    {"platform": "docker", "url": site_url, "profile_name": p}, "SOCIAL", parent=event
+                    {"platform": "docker", "url": site_url, "profile_name": p},
+                    "SOCIAL",
+                    parent=event,
+                    context=f"{{module}} tried {event.type} {event.data} and found docker profile ({{event.type}}) at {p}",
                 )
 
     async def handle_social(self, event):
@@ -48,11 +51,24 @@ async def handle_social(self, event):
         if not username:
             return
         # emit API endpoint to be visited by httpx (for url/email extraction, etc.)
-        await self.emit_event(f"{self.api_url}/users/{username}", "URL_UNVERIFIED", parent=event, tags="httpx-safe")
+        profile_url = f"{self.api_url}/users/{username}"
+        await self.emit_event(
+            profile_url,
+            "URL_UNVERIFIED",
+            parent=event,
+            tags="httpx-safe",
+            context=f"{{module}} produced {{event.type}}: {profile_url}",
+        )
         self.verbose(f"Searching for docker images belonging to {username}")
         repos = await self.get_repos(username)
         for repo in repos:
-            await self.emit_event({"url": repo}, "CODE_REPOSITORY", tags="docker", parent=event)
+            await self.emit_event(
+                {"url": repo},
+                "CODE_REPOSITORY",
+                tags="docker",
+                parent=event,
+                context=f"{{module}} found docker image {{event.type}}: {repo}",
+            )
 
     async def get_repos(self, username):
         repos = []
diff --git a/bbot/modules/git.py b/bbot/modules/git.py
index e0cf47314..98343abbb 100644
--- a/bbot/modules/git.py
+++ b/bbot/modules/git.py
@@ -24,19 +24,16 @@ async def handle_event(self, event):
             self.helpers.urljoin(base_url, ".git/config"),
             self.helpers.urljoin(f"{base_url}/", ".git/config"),
         }
-        tasks = [self.get_url(u) for u in urls]
-        async for task in self.helpers.as_completed(tasks):
-            result, url = await task
-            text = getattr(result, "text", "")
+        async for url, response in self.helpers.request_batch(urls):
+            text = getattr(response, "text", "")
             if not text:
                 text = ""
             if text:
-                if getattr(result, "status_code", 0) == 200 and "[core]" in text and not self.fp_regex.match(text):
+                if getattr(response, "status_code", 0) == 200 and "[core]" in text and not self.fp_regex.match(text):
+                    description = f"Exposed .git config at {url}"
                     await self.emit_event(
-                        {"host": str(event.host), "url": url, "description": f"Exposed .git config at {url}"},
+                        {"host": str(event.host), "url": url, "description": description},
                         "FINDING",
                         event,
+                        context="git detected {event.type}: {description}",
                     )
-
-    async def get_url(self, url):
-        return (await self.helpers.request(url), url)
diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py
index 39c46b5c1..7d678ef7b 100644
--- a/bbot/modules/github_codesearch.py
+++ b/bbot/modules/github_codesearch.py
@@ -27,13 +27,16 @@ async def handle_event(self, event):
             repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", tags="git", parent=event)
             if repo_event is None:
                 continue
-            await self.emit_event(repo_event)
+            await self.emit_event(
+                repo_event,
+                context=f'{{module}} searched github.com for "{query}" and found {{event.type}} with matching content at {repo_url}',
+            )
             for raw_url in raw_urls:
                 url_event = self.make_event(raw_url, "URL_UNVERIFIED", parent=repo_event, tags=["httpx-safe"])
                 if not url_event:
                     continue
                 url_event.scope_distance = repo_event.scope_distance
-                await self.emit_event(url_event)
+                await self.emit_event(url_event, f'file matching query "{query}" is at {{event.type}}: {raw_url}')
 
     async def query(self, query):
         repos = {}
diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py
index 87077a1b2..1d115b925 100644
--- a/bbot/modules/github_org.py
+++ b/bbot/modules/github_org.py
@@ -63,17 +63,24 @@ async def handle_event(self, event):
                 if not repo_event:
                     continue
                 repo_event.scope_distance = event.scope_distance
-                await self.emit_event(repo_event)
+                await self.emit_event(
+                    repo_event,
+                    context=f"{{module}} listed repos for GitHub profile and discovered {{event.type}}: {repo_url}",
+                )
 
             # find members from org (SOCIAL --> SOCIAL)
             if is_org and self.include_members:
                 self.verbose(f"Searching for any members belonging to {user}")
                 org_members = await self.query_org_members(user)
                 for member in org_members:
-                    event_data = {"platform": "github", "profile_name": member, "url": f"https://github.com/{member}"}
+                    member_url = f"https://github.com/{member}"
+                    event_data = {"platform": "github", "profile_name": member, "url": member_url}
                     member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", parent=event)
                     if member_event:
-                        await self.emit_event(member_event)
+                        await self.emit_event(
+                            member_event,
+                            context=f"{{module}} listed members of GitHub organization and discovered {{event.type}}: {member_url}",
+                        )
 
         # find valid orgs from stub (ORG_STUB --> SOCIAL)
         elif event.type == "ORG_STUB":
@@ -86,11 +93,15 @@ async def handle_event(self, event):
                 self.verbose(f"Unable to validate that {user} is in-scope, skipping...")
                 return
 
-            event_data = {"platform": "github", "profile_name": user, "url": f"https://github.com/{user}"}
+            user_url = f"https://github.com/{user}"
+            event_data = {"platform": "github", "profile_name": user, "url": user_url}
             github_org_event = self.make_event(event_data, "SOCIAL", tags="github-org", parent=event)
             if github_org_event:
                 github_org_event.scope_distance = event.scope_distance
-                await self.emit_event(github_org_event)
+                await self.emit_event(
+                    github_org_event,
+                    context=f'{{module}} tried "{user}" as GitHub profile and discovered {{event.type}}: {user_url}',
+                )
 
     async def query_org_repos(self, query):
         repos = []
diff --git a/bbot/modules/gitlab.py b/bbot/modules/gitlab.py
index 0d96fea8e..3404f3ba3 100644
--- a/bbot/modules/gitlab.py
+++ b/bbot/modules/gitlab.py
@@ -53,12 +53,17 @@ async def handle_http_response(self, event):
         if "x_gitlab_meta" in headers:
             url = event.parsed_url._replace(path="/").geturl()
             await self.emit_event(
-                {"host": str(event.host), "technology": "GitLab", "url": url}, "TECHNOLOGY", parent=event
+                {"host": str(event.host), "technology": "GitLab", "url": url},
+                "TECHNOLOGY",
+                parent=event,
+                context=f"{{module}} detected {{event.type}}: GitLab at {url}",
             )
+            description = f"GitLab server at {event.host}"
             await self.emit_event(
-                {"host": str(event.host), "description": f"GitLab server at {event.host}"},
+                {"host": str(event.host), "description": description},
                 "FINDING",
                 parent=event,
+                context=f"{{module}} detected {{event.type}}: {description}",
             )
 
     async def handle_technology(self, event):
@@ -95,7 +100,9 @@ async def handle_projects_url(self, projects_url, event):
             if project_url:
                 code_event = self.make_event({"url": project_url}, "CODE_REPOSITORY", tags="git", parent=event)
                 code_event.scope_distance = event.scope_distance
-                await self.emit_event(code_event)
+                await self.emit_event(
+                    code_event, context=f"{{module}} enumerated projects and found {{event.type}} at {project_url}"
+                )
             namespace = project.get("namespace", {})
             if namespace:
                 await self.handle_namespace(namespace, event)
@@ -127,7 +134,10 @@ async def handle_namespace(self, namespace, event):
                 parent=event,
             )
             social_event.scope_distance = event.scope_distance
-            await self.emit_event(social_event)
+            await self.emit_event(
+                social_event,
+                context=f'{{module}} found GitLab namespace ({{event.type}}) "{namespace_name}" at {namespace_url}',
+            )
 
     def get_base_url(self, event):
         base_url = event.data.get("url", "")
diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py
index 99b4ce8f1..e736bec1a 100644
--- a/bbot/modules/postman.py
+++ b/bbot/modules/postman.py
@@ -29,8 +29,8 @@ class postman(subdomain_enum):
     async def handle_event(self, event):
         query = self.make_query(event)
         self.verbose(f"Searching for any postman workspaces, collections, requests belonging to {query}")
-        for url in await self.query(query):
-            await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags="httpx-safe")
+        for url, context in await self.query(query):
+            await self.emit_event(url, "URL_UNVERIFIED", parent=event, tags="httpx-safe", context=context)
 
     async def query(self, query):
         interesting_urls = []
@@ -78,16 +78,46 @@ async def query(self, query):
             tldextract = self.helpers.tldextract(query)
             if tldextract.domain.lower() in name.lower():
                 self.verbose(f"Discovered workspace {name} ({id})")
-                interesting_urls.append(f"{self.base_url}/workspace/{id}")
+                workspace_url = f"{self.base_url}/workspace/{id}"
+                interesting_urls.append(
+                    (
+                        workspace_url,
+                        f'{{module}} searched postman.com for "{query}" and found matching workspace "{name}" at {{event.type}}: {workspace_url}',
+                    )
+                )
                 environments, collections = await self.search_workspace(id)
-                interesting_urls.append(f"{self.base_url}/workspace/{id}/globals")
+                globals_url = f"{self.base_url}/workspace/{id}/globals"
+                interesting_urls.append(
+                    (
+                        globals_url,
+                        f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, and found globals at {{event.type}}: {globals_url}',
+                    )
+                )
                 for e_id in environments:
-                    interesting_urls.append(f"{self.base_url}/environment/{e_id}")
+                    env_url = f"{self.base_url}/environment/{e_id}"
+                    interesting_urls.append(
+                        (
+                            env_url,
+                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated environments, and found {{event.type}}: {env_url}',
+                        )
+                    )
                 for c_id in collections:
-                    interesting_urls.append(f"{self.base_url}/collection/{c_id}")
+                    collection_url = f"{self.base_url}/collection/{c_id}"
+                    interesting_urls.append(
+                        (
+                            collection_url,
+                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated collections, and found {{event.type}}: {collection_url}',
+                        )
+                    )
                 requests = await self.search_collections(id)
                 for r_id in requests:
-                    interesting_urls.append(f"{self.base_url}/request/{r_id}")
+                    request_url = f"{self.base_url}/request/{r_id}"
+                    interesting_urls.append(
+                        (
+                            request_url,
+                            f'{{module}} searched postman.com for "{query}", found matching workspace "{name}" at {workspace_url}, enumerated requests, and found {{event.type}}: {request_url}',
+                        )
+                    )
             else:
                 self.verbose(f"Skipping workspace {name} ({id}) as it does not appear to be in scope")
         return interesting_urls
diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index ceb5172e7..d49480c1d 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -61,7 +61,12 @@ async def handle_event(self, event):
                 }
                 if description:
                     data["description"] += f" Description: [{description}]"
-                await self.emit_event(data, "VULNERABILITY", event)
+                await self.emit_event(
+                    data,
+                    "VULNERABILITY",
+                    event,
+                    context=f'{{module}} searched {event.type} using "{module}" method and found verified secret ({{event.type}}): {raw_result}',
+                )
             else:
                 data = {
                     "description": f"Potential Secret Found. Detector Type: [{detector_name}] Decoder Type: [{decoder_name}] Secret: [{raw_result}] Details: [{source_metadata}]",
@@ -69,7 +74,12 @@ async def handle_event(self, event):
                 }
                 if description:
                     data["description"] += f" Description: [{description}]"
-                await self.emit_event(data, "FINDING", event)
+                await self.emit_event(
+                    data,
+                    "FINDING",
+                    event,
+                    context=f'{{module}} searched {event.type} using "{module}" method and found verified secret ({{event.type}}): {raw_result}',
+                )
 
     async def execute_trufflehog(self, module, path):
         command = [

From 15cf73294ac05330f495da850c8f914ee41632b9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 01:44:14 -0400
Subject: [PATCH 185/220] steady work on event context

---
 bbot/core/helpers/misc.py             | 44 +++++++++++++++++++++++++++
 bbot/modules/ajaxpro.py               |  2 ++
 bbot/modules/azure_realm.py           |  5 ++-
 bbot/modules/azure_tenant.py          | 15 +++++++--
 bbot/modules/baddns.py                | 18 +++++++++--
 bbot/modules/badsecrets.py            | 18 +++++++++--
 bbot/modules/bevigil.py               | 14 +++++++--
 bbot/modules/bucket_file_enum.py      |  9 +++++-
 bbot/modules/builtwith.py             | 15 +++++++--
 bbot/modules/bypass403.py             |  2 ++
 bbot/modules/code_repository.py       |  6 +++-
 bbot/modules/credshed.py              | 34 ++++++++++++++++-----
 bbot/modules/dehashed.py              | 42 +++++++++++++++++++------
 bbot/modules/dnsbrute.py              | 10 ++++--
 bbot/modules/dnsbrute_mutations.py    | 16 +++++-----
 bbot/modules/dnscommonsrv.py          | 10 ++++--
 bbot/modules/docker_pull.py           |  4 ++-
 bbot/modules/dockerhub.py             |  9 ------
 bbot/modules/dotnetnuke.py            | 28 ++++++++++++-----
 bbot/modules/emailformat.py           |  7 ++++-
 bbot/modules/ffuf_shortnames.py       | 40 +++++++++++++++++++-----
 bbot/modules/fingerprintx.py          | 28 ++++++++++-------
 bbot/modules/generic_ssrf.py          |  1 +
 bbot/modules/git.py                   |  2 +-
 bbot/modules/git_clone.py             |  5 ++-
 bbot/modules/github_codesearch.py     |  4 ++-
 bbot/modules/github_workflows.py      |  8 +++--
 bbot/modules/gowitness.py             | 22 ++++++++++++--
 bbot/modules/host_header.py           | 27 ++++++++++------
 bbot/modules/hunt.py                  |  7 ++++-
 bbot/modules/hunterio.py              | 19 ++++++++++--
 bbot/modules/iis_shortnames.py        |  9 +++++-
 bbot/modules/internal/speculate.py    | 11 +++++--
 bbot/modules/internetdb.py            | 31 +++++++++++++++----
 bbot/modules/ip2location.py           | 13 +++++++-
 bbot/modules/ipneighbor.py            |  5 ++-
 bbot/modules/ipstack.py               | 13 +++++++-
 bbot/modules/newsletters.py           |  7 ++++-
 bbot/modules/ntlm.py                  | 23 +++++++-------
 bbot/modules/oauth.py                 | 26 +++++++++++++---
 bbot/modules/paramminer_headers.py    |  2 ++
 bbot/test/test_step_1/test_helpers.py | 17 +++++++++++
 42 files changed, 495 insertions(+), 133 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 09d2870d4..4f16dd2ce 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -2052,6 +2052,50 @@ def human_to_bytes(filesize):
     raise ValueError(f'Unable to convert filesize "{filesize}" to bytes')
 
 
+def integer_to_ordinal(n):
+    """
+    Convert an integer to its ordinal representation.
+
+    Args:
+        n (int): The integer to convert.
+
+    Returns:
+        str: The ordinal representation of the integer.
+
+    Examples:
+        >>> integer_to_ordinal(1)
+        '1st'
+        >>> integer_to_ordinal(2)
+        '2nd'
+        >>> integer_to_ordinal(3)
+        '3rd'
+        >>> integer_to_ordinal(11)
+        '11th'
+        >>> integer_to_ordinal(21)
+        '21st'
+        >>> integer_to_ordinal(101)
+        '101st'
+    """
+    # Check the last digit
+    last_digit = n % 10
+    # Check the last two digits for special cases (11th, 12th, 13th)
+    last_two_digits = n % 100
+
+    if 10 <= last_two_digits <= 20:
+        suffix = "th"
+    else:
+        if last_digit == 1:
+            suffix = "st"
+        elif last_digit == 2:
+            suffix = "nd"
+        elif last_digit == 3:
+            suffix = "rd"
+        else:
+            suffix = "th"
+
+    return f"{n}{suffix}"
+
+
 def cpu_architecture():
     """Return the CPU architecture of the current system.
 
diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py
index 3d8467a80..dda98ad2b 100644
--- a/bbot/modules/ajaxpro.py
+++ b/bbot/modules/ajaxpro.py
@@ -37,6 +37,7 @@ async def handle_event(self, event):
                                     },
                                     "FINDING",
                                     event,
+                                    context="{module} discovered Ajaxpro instance ({event.type}) at {event.data}",
                                 )
 
         elif event.type == "HTTP_RESPONSE":
@@ -53,4 +54,5 @@ async def handle_event(self, event):
                         },
                         "FINDING",
                         event,
+                        context="{module} discovered Ajaxpro instance ({event.type}) at {event.data}",
                     )
diff --git a/bbot/modules/azure_realm.py b/bbot/modules/azure_realm.py
index 0cbb806bc..9b09eaf5f 100644
--- a/bbot/modules/azure_realm.py
+++ b/bbot/modules/azure_realm.py
@@ -26,7 +26,10 @@ async def handle_event(self, event):
                     auth_url, "URL_UNVERIFIED", parent=event, tags=["affiliate", "ms-auth-url"]
                 )
                 url_event.source_domain = domain
-                await self.emit_event(url_event)
+                await self.emit_event(
+                    url_event,
+                    context="{module} queried login.microsoftonline.com for user realm and found {event.type}: {event.data}",
+                )
 
     async def getuserrealm(self, domain):
         url = f"https://login.microsoftonline.com/getuserrealm.srf?login=test@{domain}"
diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index 4c2ee63de..76b2e7f0a 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -38,7 +38,13 @@ async def handle_event(self, event):
             self.verbose(f'Found {len(domains):,} domains under tenant for "{query}": {", ".join(sorted(domains))}')
             for domain in domains:
                 if domain != query:
-                    await self.emit_event(domain, "DNS_NAME", parent=event, tags=["affiliate", "azure-tenant"])
+                    await self.emit_event(
+                        domain,
+                        "DNS_NAME",
+                        parent=event,
+                        tags=["affiliate", "azure-tenant"],
+                        context=f'{{module}} queried Outlook autodiscover for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
                     # tenant names
                     if domain.lower().endswith(".onmicrosoft.com"):
                         tenantname = domain.split(".")[0].lower()
@@ -48,7 +54,12 @@ async def handle_event(self, event):
             event_data = {"tenant-names": sorted(tenant_names), "domains": sorted(domains)}
             if tenant_id is not None:
                 event_data["tenant-id"] = tenant_id
-            await self.emit_event(event_data, "AZURE_TENANT", parent=event)
+            await self.emit_event(
+                event_data,
+                "AZURE_TENANT",
+                parent=event,
+                context=f'{{module}} queried Outlook autodiscover for "{query}" and found {{event.type}}: {{event.data}}',
+            )
 
     async def query(self, domain):
         url = f"{self.base_url}/autodiscover/autodiscover.svc"
diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py
index eafac9861..19c0a287d 100644
--- a/bbot/modules/baddns.py
+++ b/bbot/modules/baddns.py
@@ -66,7 +66,11 @@ async def handle_event(self, event):
                                 "host": str(event.host),
                             }
                             await self.emit_event(
-                                data, "VULNERABILITY", event, tags=[f"baddns-{module_instance.name.lower()}"]
+                                data,
+                                "VULNERABILITY",
+                                event,
+                                tags=[f"baddns-{module_instance.name.lower()}"],
+                                context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
                             )
 
                         elif r_dict["confidence"] in ["UNLIKELY", "POSSIBLE"] and not self.only_high_confidence:
@@ -75,7 +79,11 @@ async def handle_event(self, event):
                                 "host": str(event.host),
                             }
                             await self.emit_event(
-                                data, "FINDING", event, tags=[f"baddns-{module_instance.name.lower()}"]
+                                data,
+                                "FINDING",
+                                event,
+                                tags=[f"baddns-{module_instance.name.lower()}"],
+                                context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {r_dict["description"]}',
                             )
                         else:
                             self.warning(f"Got unrecognized confidence level: {r['confidence']}")
@@ -84,5 +92,9 @@ async def handle_event(self, event):
                         if found_domains:
                             for found_domain in found_domains:
                                 await self.emit_event(
-                                    found_domain, "DNS_NAME", event, tags=[f"baddns-{module_instance.name.lower()}"]
+                                    found_domain,
+                                    "DNS_NAME",
+                                    event,
+                                    tags=[f"baddns-{module_instance.name.lower()}"],
+                                    context=f'{{module}}\'s "{r_dict["module"]}" module found {{event.type}}: {{event.data}}',
                                 )
diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py
index 7153e78ba..8f8201ada 100644
--- a/bbot/modules/badsecrets.py
+++ b/bbot/modules/badsecrets.py
@@ -72,14 +72,21 @@ async def handle_event(self, event):
                             "url": event.data["url"],
                             "host": str(event.host),
                         }
-                        await self.emit_event(data, "VULNERABILITY", event)
+                        await self.emit_event(
+                            data,
+                            "VULNERABILITY",
+                            event,
+                            context=f'{{module}}\'s "{r["detecting_module"]}" module found known {r["description"]["product"]} secret ({{event.type}}): "{r["secret"]}"',
+                        )
                     elif r["type"] == "IdentifyOnly":
                         # There is little value to presenting a non-vulnerable asp.net viewstate, as it is not crackable without a Matrioshka brain. Just emit a technology instead.
                         if r["detecting_module"] == "ASPNET_Viewstate":
+                            technology = "microsoft asp.net"
                             await self.emit_event(
-                                {"technology": "microsoft asp.net", "url": event.data["url"], "host": str(event.host)},
+                                {"technology": technology, "url": event.data["url"], "host": str(event.host)},
                                 "TECHNOLOGY",
                                 event,
+                                context=f"{{module}} identified {{event.type}}: {technology}",
                             )
                         else:
                             data = {
@@ -87,4 +94,9 @@ async def handle_event(self, event):
                                 "url": event.data["url"],
                                 "host": str(event.host),
                             }
-                            await self.emit_event(data, "FINDING", event)
+                            await self.emit_event(
+                                data,
+                                "FINDING",
+                                event,
+                                context=f'{{module}} identified cryptographic product ({{event.type}}): "{r["description"]["product"]}"',
+                            )
diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py
index 69a671294..3926d90b3 100644
--- a/bbot/modules/bevigil.py
+++ b/bbot/modules/bevigil.py
@@ -34,13 +34,23 @@ async def handle_event(self, event):
         subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains)
         if subdomains:
             for subdomain in subdomains:
-                await self.emit_event(subdomain, "DNS_NAME", parent=event)
+                await self.emit_event(
+                    subdomain,
+                    "DNS_NAME",
+                    parent=event,
+                    context=f'{{module}} queried BeVigil\'s API for "{query}" and discovered {{event.type}}: {{event.data}}',
+                )
 
         if self.urls:
             urls = await self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls)
             if urls:
                 for parsed_url in await self.helpers.run_in_executor_mp(self.helpers.validators.collapse_urls, urls):
-                    await self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", parent=event)
+                    await self.emit_event(
+                        parsed_url.geturl(),
+                        "URL_UNVERIFIED",
+                        parent=event,
+                        context=f'{{module}} queried BeVigil\'s API for "{query}" and discovered {{event.type}}: {{event.data}}',
+                    )
 
     async def request_subdomains(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/"
diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py
index 2e7da60d3..a8c06996a 100644
--- a/bbot/modules/bucket_file_enum.py
+++ b/bbot/modules/bucket_file_enum.py
@@ -44,7 +44,14 @@ async def handle_aws(self, event):
                 bucket_file = url + "/" + key
                 file_extension = self.helpers.get_file_extension(key)
                 if file_extension not in self.scan.url_extension_blacklist:
-                    await self.emit_event(bucket_file, "URL_UNVERIFIED", parent=event, tags="filedownload")
+                    extension_upper = file_extension.upper()
+                    await self.emit_event(
+                        bucket_file,
+                        "URL_UNVERIFIED",
+                        parent=event,
+                        tags="filedownload",
+                        context=f"{{module}} enumerate files in bucket and discovered {extension_upper} file at {{event.type}}: {{event.data}}",
+                    )
                     urls_emitted += 1
                     if urls_emitted >= self.file_limit:
                         return
diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py
index bf3b9a728..e51bb5db8 100644
--- a/bbot/modules/builtwith.py
+++ b/bbot/modules/builtwith.py
@@ -38,14 +38,25 @@ async def handle_event(self, event):
         if subdomains:
             for s in subdomains:
                 if s != event:
-                    await self.emit_event(s, "DNS_NAME", parent=event)
+                    await self.emit_event(
+                        s,
+                        "DNS_NAME",
+                        parent=event,
+                        context=f'{{module}} queried the BuiltWith API for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
         # redirects
         if self.config.get("redirects", True):
             redirects = await self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects)
             if redirects:
                 for r in redirects:
                     if r != event:
-                        await self.emit_event(r, "DNS_NAME", parent=event, tags=["affiliate"])
+                        await self.emit_event(
+                            r,
+                            "DNS_NAME",
+                            parent=event,
+                            tags=["affiliate"],
+                            context=f'{{module}} queried the BuiltWith redirect API for "{query}" and found redirect to {{event.type}}: {{event.data}}',
+                        )
 
     async def request_domains(self, query):
         url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes"
diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index c66ba72a8..38e8b570f 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -147,6 +147,7 @@ async def handle_event(self, event):
                 },
                 "FINDING",
                 parent=event,
+                context=f"{{module}} discovered multiple 403 bypasses ({{event.type}}) for {event.data}",
             )
         else:
             for description in results:
@@ -154,6 +155,7 @@ async def handle_event(self, event):
                     {"description": description, "host": str(event.host), "url": event.data},
                     "FINDING",
                     parent=event,
+                    context=f"{{module}} discovered 403 bypass ({{event.type}}) for {event.data}",
                 )
 
     # When a WAF-check helper is available in the future, we will convert to HTTP_RESPONSE and check for the WAF string here.
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
index 2407172bc..f37c80407 100644
--- a/bbot/modules/code_repository.py
+++ b/bbot/modules/code_repository.py
@@ -50,4 +50,8 @@ async def handle_event(self, event):
                         parent=event,
                     )
                     repo_event.scope_distance = event.scope_distance
-                    await self.emit_event(repo_event, context=f"detected {{event.type}}: {url}")
+                    await self.emit_event(
+                        repo_event,
+                        context=f"detected {{event.type}}: {url}",
+                        context=f"{{module}} detected {platform} {{event.type}} at {url}",
+                    )
diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py
index 83b51e39b..3630646a6 100644
--- a/bbot/modules/credshed.py
+++ b/bbot/modules/credshed.py
@@ -1,9 +1,9 @@
 from contextlib import suppress
 
-from bbot.modules.base import BaseModule
+from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
-class credshed(BaseModule):
+class credshed(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME", "EMAIL_ADDRESS"]
     flags = ["passive", "safe"]
@@ -43,7 +43,7 @@ async def setup(self):
         return await super().setup()
 
     async def handle_event(self, event):
-        query = event.data
+        query = self.make_query(event)
         cs_query = await self.helpers.request(
             f"{self.base_url}/api/search",
             method="POST",
@@ -79,11 +79,31 @@ async def handle_event(self, event):
 
             email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=tags)
             if email_event is not None:
-                await self.emit_event(email_event)
+                await self.emit_event(
+                    email_event, context=f'{{module}} searched for "{query}" and found {{event.type}}: {{event.data}}'
+                )
                 if user:
-                    await self.emit_event(f"{email}:{user}", "USERNAME", parent=email_event, tags=tags)
+                    await self.emit_event(
+                        f"{email}:{user}",
+                        "USERNAME",
+                        parent=email_event,
+                        tags=tags,
+                        context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
+                    )
                 if pw:
-                    await self.emit_event(f"{email}:{pw}", "PASSWORD", parent=email_event, tags=tags)
+                    await self.emit_event(
+                        f"{email}:{pw}",
+                        "PASSWORD",
+                        parent=email_event,
+                        tags=tags,
+                        context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
+                    )
                 for h_pw in hashes:
                     if h_pw:
-                        await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", parent=email_event, tags=tags)
+                        await self.emit_event(
+                            f"{email}:{h_pw}",
+                            "HASHED_PASSWORD",
+                            parent=email_event,
+                            tags=tags,
+                            context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
+                        )
diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py
index 8eda9968e..37202fbae 100644
--- a/bbot/modules/dehashed.py
+++ b/bbot/modules/dehashed.py
@@ -1,9 +1,9 @@
 from contextlib import suppress
 
-from bbot.modules.base import BaseModule
+from bbot.modules.templates.subdomain_enum import subdomain_enum
 
 
-class dehashed(BaseModule):
+class dehashed(subdomain_enum):
     watched_events = ["DNS_NAME"]
     produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"]
     flags = ["passive", "safe", "email-enum"]
@@ -34,7 +34,8 @@ async def setup(self):
         return await super().setup()
 
     async def handle_event(self, event):
-        async for entries in self.query(event):
+        query = self.make_query(event)
+        async for entries in self.query(query):
             for entry in entries:
                 # we have to clean up the email field because dehashed does a poor job of it
                 email_str = entry.get("email", "").replace("\\", "")
@@ -55,16 +56,37 @@ async def handle_event(self, event):
                 if email:
                     email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=tags)
                     if email_event is not None:
-                        await self.emit_event(email_event)
+                        await self.emit_event(
+                            email_event,
+                            context=f'{{module}} searched API for "{query}" and found {{event.type}}: {{event.data}}',
+                        )
                         if user:
-                            await self.emit_event(f"{email}:{user}", "USERNAME", parent=email_event, tags=tags)
+                            await self.emit_event(
+                                f"{email}:{user}",
+                                "USERNAME",
+                                parent=email_event,
+                                tags=tags,
+                                context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
+                            )
                         if pw:
-                            await self.emit_event(f"{email}:{pw}", "PASSWORD", parent=email_event, tags=tags)
+                            await self.emit_event(
+                                f"{email}:{pw}",
+                                "PASSWORD",
+                                parent=email_event,
+                                tags=tags,
+                                context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
+                            )
                         if h_pw:
-                            await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", parent=email_event, tags=tags)
+                            await self.emit_event(
+                                f"{email}:{h_pw}",
+                                "HASHED_PASSWORD",
+                                parent=email_event,
+                                tags=tags,
+                                context=f"{{module}} found {email} with {{event.type}}: {{event.data}}",
+                            )
 
-    async def query(self, event):
-        query = f"domain:{event.data}"
+    async def query(self, domain):
+        query = f"domain:{domain}"
         url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}"
         page = 0
         num_entries = 0
@@ -86,7 +108,7 @@ async def query(self, event):
                     )
                 elif (page >= 3) and (total > num_entries):
                     self.info(
-                        f"{event.data} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results."
+                        f"{domain} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results."
                     )
                 agen.aclose()
                 break
diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py
index 4d4240b6f..8ed05e2f0 100644
--- a/bbot/modules/dnsbrute.py
+++ b/bbot/modules/dnsbrute.py
@@ -23,6 +23,7 @@ async def setup(self):
         self.max_depth = max(1, self.config.get("max_depth", 5))
         self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
         self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
+        self.wordlist_size = len(self.subdomain_list)
         return await super().setup()
 
     async def filter_event(self, event):
@@ -44,6 +45,11 @@ async def filter_event(self, event):
 
     async def handle_event(self, event):
         query = self.make_query(event)
-        self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
+        self.info(f"Brute-forcing {self.wordlist_size:,} subdomains for {query} (source: {event.data})")
         for hostname in await self.helpers.dns.brute(self, query, self.subdomain_list):
-            await self.emit_event(hostname, "DNS_NAME", parent=event)
+            await self.emit_event(
+                hostname,
+                "DNS_NAME",
+                parent=event,
+                context=f'{{module}} tried {self.wordlist_size:,} subdomains against "{query}" and found {{event.type}}: {{event.data}}',
+            )
diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py
index 1a836c971..562fc88d8 100644
--- a/bbot/modules/dnsbrute_mutations.py
+++ b/bbot/modules/dnsbrute_mutations.py
@@ -21,7 +21,7 @@ async def setup(self):
         self.max_mutations = self.config.get("max_mutations", 500)
         # 800M bits == 100MB bloom filter == 10M entries before false positives start emerging
         self.mutations_tried = self.helpers.bloom_filter(800000000)
-        self._mutation_run = 1
+        self._mutation_run_counter = {}
         return True
 
     async def handle_event(self, event):
@@ -55,7 +55,6 @@ async def finish(self):
                     )
 
         base_mutations = set()
-        found_mutations = False
         try:
             for i, (domain, subdomains) in enumerate(trimmed_found):
                 self.verbose(f"{domain} has {len(subdomains):,} subdomains")
@@ -103,6 +102,11 @@ def add_mutation(m):
                     if mutations:
                         self.info(f"Trying {len(mutations):,} mutations against {domain} ({i+1}/{len(trimmed_found)})")
                         results = await self.helpers.dns.brute(self, query, mutations)
+                        try:
+                            mutation_run = self._mutation_run_counter[domain]
+                        except KeyError:
+                            self._mutation_run_counter[domain] = mutation_run = 1
+                        self._mutation_run_counter[domain] += 1
                         for hostname in results:
                             parent_event = self.parent_events.get_host(hostname)
                             if parent_event is None:
@@ -113,23 +117,21 @@ def add_mutation(m):
                                     self.parent_events._radix.dns_tree.root.children["com"].children["tesla"].children
                                 )
                                 parent_event = self.scan.root_event
+                            mutation_run_ordinal = self.helpers.integer_to_ordinal(mutation_run)
                             await self.emit_event(
                                 hostname,
                                 "DNS_NAME",
                                 parent=parent_event,
-                                tags=[f"mutation-{self._mutation_run}"],
+                                tags=[f"mutation-{mutation_run}"],
                                 abort_if=self.abort_if,
+                                context=f'{{module}} found a mutated subdomain of "{domain}" on its {mutation_run_ordinal} run: {{event.type}}: {{event.data}}',
                             )
                         if results:
-                            found_mutations = True
                             continue
                     break
         except AssertionError as e:
             self.warning(e)
 
-        if found_mutations:
-            self._mutation_run += 1
-
     def abort_if(self, event):
         if not event.scope_distance == 0:
             return True, "event is not in scope"
diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py
index a6b8250eb..790ff8e65 100644
--- a/bbot/modules/dnscommonsrv.py
+++ b/bbot/modules/dnscommonsrv.py
@@ -147,6 +147,7 @@
     "_imap",  # 1
     "_iax",  # 1
 ]
+num_srvs = len(common_srvs)
 
 
 class dnscommonsrv(subdomain_enum):
@@ -158,6 +159,11 @@ class dnscommonsrv(subdomain_enum):
 
     async def handle_event(self, event):
         query = self.make_query(event)
-        self.verbose(f'Brute-forcing SRV records for "{query}"')
+        self.verbose(f'Brute-forcing {num_srvs:,} SRV records for "{query}"')
         for hostname in await self.helpers.dns.brute(self, query, common_srvs, type="SRV"):
-            await self.emit_event(hostname, "DNS_NAME", parent=event)
+            await self.emit_event(
+                hostname,
+                "DNS_NAME",
+                parent=event,
+                context=f'{{module}} tried {num_srvs:,} common SRV records against "{query}" and found {{event.type}}: {{event.data}}',
+            )
diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py
index 807306a87..0ddd6578b 100644
--- a/bbot/modules/docker_pull.py
+++ b/bbot/modules/docker_pull.py
@@ -60,7 +60,9 @@ async def handle_event(self, event):
                 parent=event,
             )
             codebase_event.scope_distance = event.scope_distance
-            await self.emit_event(codebase_event)
+            await self.emit_event(
+                codebase_event, context=f"{{module}} downloaded Docker image to {{event.type}}: {repo_path}"
+            )
 
     def get_registry_and_repository(self, repository_url):
         """Function to get the registry and repository from a html repository URL."""
diff --git a/bbot/modules/dockerhub.py b/bbot/modules/dockerhub.py
index 41bc027de..c9c206d7a 100644
--- a/bbot/modules/dockerhub.py
+++ b/bbot/modules/dockerhub.py
@@ -50,15 +50,6 @@ async def handle_social(self, event):
         username = event.data.get("profile_name", "")
         if not username:
             return
-        # emit API endpoint to be visited by httpx (for url/email extraction, etc.)
-        profile_url = f"{self.api_url}/users/{username}"
-        await self.emit_event(
-            profile_url,
-            "URL_UNVERIFIED",
-            parent=event,
-            tags="httpx-safe",
-            context=f"{{module}} produced {{event.type}}: {profile_url}",
-        )
         self.verbose(f"Searching for docker images belonging to {username}")
         repos = await self.get_repos(username)
         for repo in repos:
diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index 25bdf09b9..9dfc48790 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -48,15 +48,18 @@ async def interactsh_callback(self, r):
                 event = self.interactsh_subdomain_tags.get(full_id.split(".")[0])
                 if not event:
                     return
+                url = event.data["url"]
+                description = "DotNetNuke Blind-SSRF (CVE 2017-0929)"
                 await self.emit_event(
                     {
                         "severity": "MEDIUM",
                         "host": str(event.host),
-                        "url": event.data["url"],
-                        "description": f"DotNetNuke Blind-SSRF (CVE 2017-0929)",
+                        "url": url,
+                        "description": description,
                     },
                     "VULNERABILITY",
                     event,
+                    context=f"{{module}} scanned {url} and found medium {{event.type}}: {description}",
                 )
             else:
                 # this is likely caused by something trying to resolve the base domain first and can be ignored
@@ -69,10 +72,12 @@ async def handle_event(self, event):
         if raw_headers:
             for header_signature in self.DNN_signatures_header:
                 if header_signature in raw_headers:
+                    url = event.data["url"]
                     await self.emit_event(
-                        {"technology": "DotNetNuke", "url": event.data["url"], "host": str(event.host)},
+                        {"technology": "DotNetNuke", "url": url, "host": str(event.host)},
                         "TECHNOLOGY",
                         event,
+                        context=f"{{module}} scanned {url} and found {{event.type}}: DotNetNuke",
                     )
                     detected = True
                     break
@@ -84,6 +89,7 @@ async def handle_event(self, event):
                         {"technology": "DotNetNuke", "url": event.data["url"], "host": str(event.host)},
                         "TECHNOLOGY",
                         event,
+                        context=f"{{module}} scanned {url} and found {{event.type}}: DotNetNuke",
                     )
                     detected = True
                     break
@@ -94,15 +100,17 @@ async def handle_event(self, event):
                 result = await self.helpers.request(probe_url, cookies=self.exploit_probe)
                 if result:
                     if "for 16-bit app support" in result.text and "[extensions]" in result.text:
+                        description = "DotNetNuke Personalization Cookie Deserialization"
                         await self.emit_event(
                             {
                                 "severity": "CRITICAL",
-                                "description": "DotNetNuke Personalization Cookie Deserialization",
+                                "description": description,
                                 "host": str(event.host),
                                 "url": probe_url,
                             },
                             "VULNERABILITY",
                             event,
+                            context=f"{{module}} scanned {probe_url} and found critical {{event.type}}: {description}",
                         )
 
             if "endpoint" not in event.tags:
@@ -113,15 +121,17 @@ async def handle_event(self, event):
                 )
                 if result:
                     if "<configuration>" in result.text:
+                        description = "DotNetNuke dnnUI_NewsArticlesSlider Module Arbitrary File Read"
                         await self.emit_event(
                             {
                                 "severity": "CRITICAL",
-                                "description": "DotNetNuke dnnUI_NewsArticlesSlider Module Arbitrary File Read",
+                                "description": description,
                                 "host": str(event.host),
                                 "url": f'{event.data["url"]}/DesktopModules/dnnUI_NewsArticlesSlider/ImageHandler.ashx',
                             },
                             "VULNERABILITY",
                             event,
+                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
                         )
 
                 # DNNArticle GetCSS.ashx File Read
@@ -130,15 +140,17 @@ async def handle_event(self, event):
                 )
                 if result:
                     if "<configuration>" in result.text:
+                        description = "DotNetNuke DNNArticle Module GetCSS.ashx Arbitrary File Read"
                         await self.emit_event(
                             {
                                 "severity": "CRITICAL",
-                                "description": "DotNetNuke DNNArticle Module GetCSS.ashx Arbitrary File Read",
+                                "description": description,
                                 "host": str(event.host),
                                 "url": f'{event.data["url"]}/Desktopmodules/DNNArticle/GetCSS.ashx/?CP=%2fweb.config',
                             },
                             "VULNERABILITY",
                             event,
+                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
                         )
 
                 # InstallWizard SuperUser Privilege Escalation
@@ -148,15 +160,17 @@ async def handle_event(self, event):
                         f'{event.data["url"]}/Install/InstallWizard.aspx?__viewstate=1'
                     )
                     if result_confirm.status_code == 500:
+                        description = "DotNetNuke InstallWizard SuperUser Privilege Escalation"
                         await self.emit_event(
                             {
                                 "severity": "CRITICAL",
-                                "description": "DotNetNuke InstallWizard SuperUser Privilege Escalation",
+                                "description": description,
                                 "host": str(event.host),
                                 "url": f'{event.data["url"]}/Install/InstallWizard.aspx',
                             },
                             "VULNERABILITY",
                             event,
+                            context=f'{{module}} scanned {event.data["url"]} and found critical {{event.type}}: {description}',
                         )
                         return
 
diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py
index 6ffbc6b90..c7161070a 100644
--- a/bbot/modules/emailformat.py
+++ b/bbot/modules/emailformat.py
@@ -23,4 +23,9 @@ async def handle_event(self, event):
             return
         for email in await self.helpers.re.extract_emails(r.text):
             if email.endswith(query):
-                await self.emit_event(email, "EMAIL_ADDRESS", parent=event)
+                await self.emit_event(
+                    email,
+                    "EMAIL_ADDRESS",
+                    parent=event,
+                    context=f'{{module}} searched email-format.com for "{query}" and found {{event.type}}: {{event.data}}',
+                )
diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py
index c09e0695a..6fcb90f0f 100644
--- a/bbot/modules/ffuf_shortnames.py
+++ b/bbot/modules/ffuf_shortnames.py
@@ -152,12 +152,24 @@ async def handle_event(self, event):
             if "shortname-file" in event.tags:
                 for ext in used_extensions:
                     async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"):
-                        await self.emit_event(r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
+                        await self.emit_event(
+                            r["url"],
+                            "URL_UNVERIFIED",
+                            parent=event,
+                            tags=[f"status-{r['status']}"],
+                            context=f"{{module}} brute-forced {ext.upper()} files at {root_url} and found {{event.type}}: {{event.data}}",
+                        )
 
             elif "shortname-directory" in event.tags:
                 async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]):
                     r_url = f"{r['url'].rstrip('/')}/"
-                    await self.emit_event(r_url, "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
+                    await self.emit_event(
+                        r_url,
+                        "URL_UNVERIFIED",
+                        parent=event,
+                        tags=[f"status-{r['status']}"],
+                        context=f"{{module}} brute-forced directories at {r_url} and found {{event.type}}: {{event.data}}",
+                    )
 
         if self.config.get("find_delimiters"):
             if "shortname-directory" in event.tags:
@@ -166,8 +178,15 @@ async def handle_event(self, event):
                     delimiter, prefix, partial_hint = delimiter_r
                     self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
                     tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
-                    async for r in self.execute_ffuf(tempfile, root_url, prefix=f"{prefix}{delimiter}", exts=["/"]):
-                        await self.emit_event(r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
+                    ffuf_prefix = f"{prefix}{delimiter}"
+                    async for r in self.execute_ffuf(tempfile, root_url, prefix=ffuf_prefix, exts=["/"]):
+                        await self.emit_event(
+                            r["url"],
+                            "URL_UNVERIFIED",
+                            parent=event,
+                            tags=[f"status-{r['status']}"],
+                            context=f'{{module}} brute-forced directories with detected prefix "{ffuf_prefix}" and found {{event.type}}: {{event.data}}',
+                        )
 
             elif "shortname-file" in event.tags:
                 for ext in used_extensions:
@@ -176,11 +195,14 @@ async def handle_event(self, event):
                         delimiter, prefix, partial_hint = delimiter_r
                         self.verbose(f"Detected delimiter [{delimiter}] in hint [{filename_hint}]")
                         tempfile, tempfile_len = self.generate_templist(prefix=partial_hint)
-                        async for r in self.execute_ffuf(
-                            tempfile, root_url, prefix=f"{prefix}{delimiter}", suffix=f".{ext}"
-                        ):
+                        ffuf_prefix = f"{prefix}{delimiter}"
+                        async for r in self.execute_ffuf(tempfile, root_url, prefix=ffuf_prefix, suffix=f".{ext}"):
                             await self.emit_event(
-                                r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"]
+                                r["url"],
+                                "URL_UNVERIFIED",
+                                parent=event,
+                                tags=[f"status-{r['status']}"],
+                                context=f'{{module}} brute-forced {ext.upper()} files with detected prefix "{ffuf_prefix}" and found {{event.type}}: {{event.data}}',
                             )
 
     async def finish(self):
@@ -214,6 +236,7 @@ async def finish(self):
                                             "URL_UNVERIFIED",
                                             parent=self.shortname_to_event[hint],
                                             tags=[f"status-{r['status']}"],
+                                            context=f'{{module}} brute-forced directories with common prefix "{prefix}" and found {{event.type}}: {{event.data}}',
                                         )
                                 elif "shortname-file" in self.shortname_to_event[hint].tags:
                                     used_extensions = self.build_extension_list(self.shortname_to_event[hint])
@@ -230,4 +253,5 @@ async def finish(self):
                                                 "URL_UNVERIFIED",
                                                 parent=self.shortname_to_event[hint],
                                                 tags=[f"status-{r['status']}"],
+                                                context=f'{{module}} brute-forced {ext.upper()} files with common prefix "{prefix}" and found {{event.type}}: {{event.data}}',
                                             )
diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py
index 9e18e2fb0..cac87c5c3 100644
--- a/bbot/modules/fingerprintx.py
+++ b/bbot/modules/fingerprintx.py
@@ -42,18 +42,24 @@ async def handle_batch(self, *events):
             ip = j.get("ip", "")
             host = j.get("host", ip)
             port = str(j.get("port", ""))
+            protocol = j.get("protocol", "").upper()
+            if not host and port and protocol:
+                continue
             banner = j.get("metadata", {}).get("banner", "").strip()
-            if port:
-                port_data = f"{host}:{port}"
-            protocol = j.get("protocol", "")
+            port_data = f"{host}:{port}"
             tags = set()
             if host and ip:
                 tags.add(f"ip-{ip}")
-            if host and port and protocol:
-                parent_event = _input.get(port_data)
-                protocol_data = {"host": host, "protocol": protocol.upper()}
-                if port:
-                    protocol_data["port"] = port
-                if banner:
-                    protocol_data["banner"] = banner
-                await self.emit_event(protocol_data, "PROTOCOL", parent=parent_event, tags=tags)
+            parent_event = _input.get(port_data)
+            protocol_data = {"host": host, "protocol": protocol}
+            if port:
+                protocol_data["port"] = port
+            if banner:
+                protocol_data["banner"] = banner
+            await self.emit_event(
+                protocol_data,
+                "PROTOCOL",
+                parent=parent_event,
+                tags=tags,
+                context=f"{{module}} probed {port_data} and detected {{event.type}}: {protocol}",
+            )
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index 1aa88637e..0aa61a3e5 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -209,6 +209,7 @@ async def interactsh_callback(self, r):
                     },
                     "VULNERABILITY",
                     matched_event,
+                    context=f"{{module}} scanned {matched_event.data} and detected {{event.type}}: {matched_technique}",
                 )
             else:
                 # this is likely caused by something trying to resolve the base domain first and can be ignored
diff --git a/bbot/modules/git.py b/bbot/modules/git.py
index 98343abbb..9a180bc11 100644
--- a/bbot/modules/git.py
+++ b/bbot/modules/git.py
@@ -35,5 +35,5 @@ async def handle_event(self, event):
                         {"host": str(event.host), "url": url, "description": description},
                         "FINDING",
                         event,
-                        context="git detected {event.type}: {description}",
+                        context="{module} detected {event.type}: {description}",
                     )
diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py
index 026ee5ebf..95083ecf4 100644
--- a/bbot/modules/git_clone.py
+++ b/bbot/modules/git_clone.py
@@ -42,7 +42,10 @@ async def handle_event(self, event):
                 {"path": str(repo_path)}, "FILESYSTEM", tags=["git", "folder"], parent=event
             )
             codebase_event.scope_distance = event.scope_distance
-            await self.emit_event(codebase_event)
+            await self.emit_event(
+                codebase_event,
+                context=f"{{module}} downloaded git repo at {repo_url} to {{event.type}}: {repo_path}",
+            )
 
     async def clone_git_repository(self, repository_url):
         if self.api_key:
diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py
index 7d678ef7b..ddafb025f 100644
--- a/bbot/modules/github_codesearch.py
+++ b/bbot/modules/github_codesearch.py
@@ -36,7 +36,9 @@ async def handle_event(self, event):
                 if not url_event:
                     continue
                 url_event.scope_distance = repo_event.scope_distance
-                await self.emit_event(url_event, f'file matching query "{query}" is at {{event.type}}: {raw_url}')
+                await self.emit_event(
+                    url_event, context=f'file matching query "{query}" is at {{event.type}}: {raw_url}'
+                )
 
     async def query(self, query):
         repos = {}
diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py
index 1fb1ad406..76ed2d5ff 100644
--- a/bbot/modules/github_workflows.py
+++ b/bbot/modules/github_workflows.py
@@ -48,17 +48,21 @@ async def handle_event(self, event):
                 run_id = run.get("id")
                 self.log.debug(f"Downloading logs for {workflow_name}/{run_id} in {owner}/{repo}")
                 for log in await self.download_run_logs(owner, repo, run_id):
+                    workflow_url = f"https://github.com/{owner}/{repo}/actions/runs/{run_id}"
                     logfile_event = self.make_event(
                         {
                             "path": str(log),
-                            "description": f"Workflow run logs from https://github.com/{owner}/{repo}/actions/runs/{run_id}",
+                            "description": f"Workflow run logs from {workflow_url}",
                         },
                         "FILESYSTEM",
                         tags=["textfile"],
                         parent=event,
                     )
                     logfile_event.scope_distance = event.scope_distance
-                    await self.emit_event(logfile_event)
+                    await self.emit_event(
+                        logfile_event,
+                        context=f"{{module}} downloaded workflow run logs from {workflow_url} to {{event.type}}: {log}",
+                    )
 
     async def get_workflows(self, owner, repo):
         workflows = []
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 9abb70d64..59482ecbb 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -142,7 +142,12 @@ async def handle_batch(self, *events):
             filename = screenshot["filename"]
             webscreenshot_data = {"filename": filename, "url": final_url}
             parent_event = event_dict[url]
-            await self.emit_event(webscreenshot_data, "WEBSCREENSHOT", parent=parent_event)
+            await self.emit_event(
+                webscreenshot_data,
+                "WEBSCREENSHOT",
+                parent=parent_event,
+                context=f"{{module}} visited {final_url} and saved {{event.type}} to {filename}",
+            )
 
         # emit URLs
         for url, row in self.new_network_logs.items():
@@ -156,7 +161,13 @@ async def handle_batch(self, *events):
             if self.helpers.is_spider_danger(parent_event, url):
                 tags.append("spider-danger")
             if url and url.startswith("http"):
-                await self.emit_event(url, "URL_UNVERIFIED", parent=parent_event, tags=tags)
+                await self.emit_event(
+                    url,
+                    "URL_UNVERIFIED",
+                    parent=parent_event,
+                    tags=tags,
+                    context=f"{{module}} visited {{event.type}}: {url}",
+                )
 
         # emit technologies
         for _, row in self.new_technologies.items():
@@ -165,7 +176,12 @@ async def handle_batch(self, *events):
             parent_event = event_dict[parent_url]
             technology = row["value"]
             tech_data = {"technology": technology, "url": parent_url, "host": str(parent_event.host)}
-            await self.emit_event(tech_data, "TECHNOLOGY", parent=parent_event)
+            await self.emit_event(
+                tech_data,
+                "TECHNOLOGY",
+                parent=parent_event,
+                context=f"{{module}} visited {parent_url} and found {{event.type}}: {technology}",
+            )
 
     def construct_command(self):
         # base executable
diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py
index 60d077cdd..5e2393a47 100644
--- a/bbot/modules/host_header.py
+++ b/bbot/modules/host_header.py
@@ -44,14 +44,16 @@ async def interactsh_callback(self, r):
                 matched_event = match[0]
                 matched_technique = match[1]
 
+                protocol = r.get("protocol").upper()
                 await self.emit_event(
                     {
                         "host": str(matched_event.host),
                         "url": matched_event.data["url"],
-                        "description": f"Spoofed Host header ({matched_technique}) [{r.get('protocol').upper()}] interaction",
+                        "description": f"Spoofed Host header ({matched_technique}) [{protocol}] interaction",
                     },
                     "FINDING",
                     matched_event,
+                    context=f"{{module}} spoofed host header and induced {{event.type}}: {protocol} interaction",
                 )
             else:
                 # this is likely caused by something trying to resolve the base domain first and can be ignored
@@ -78,6 +80,7 @@ async def cleanup(self):
 
     async def handle_event(self, event):
         # get any set-cookie responses from the response and add them to the request
+        url = event.data["url"]
 
         added_cookies = {}
 
@@ -97,7 +100,7 @@ async def handle_event(self, event):
         subdomain_tag = self.rand_string(4, digits=False)
         self.subdomain_tags[subdomain_tag] = (event, technique_description)
         output = await self.helpers.curl(
-            url=event.data["url"],
+            url=url,
             headers={"Host": f"{subdomain_tag}.{self.domain}"},
             ignore_bbot_global_settings=True,
             cookies=added_cookies,
@@ -111,8 +114,8 @@ async def handle_event(self, event):
         subdomain_tag = self.rand_string(4, digits=False)
         self.subdomain_tags[subdomain_tag] = (event, technique_description)
         output = await self.helpers.curl(
-            url=event.data["url"],
-            path_override=event.data["url"],
+            url=url,
+            path_override=url,
             cookies=added_cookies,
         )
 
@@ -122,7 +125,7 @@ async def handle_event(self, event):
         # duplicate host header tolerance
         technique_description = "duplicate host header tolerance"
         output = await self.helpers.curl(
-            url=event.data["url"],
+            url=url,
             # Sending a blank HOST first as a hack to trick curl. This makes it no longer an "internal header", thereby allowing for duplicates
             # The fact that it's accepting two host headers is rare enough to note on its own, and not too noisy. Having the 3rd header be an interactsh would result in false negatives for the slightly less interesting cases.
             headers={"Host": ["", str(event.host), str(event.host)]},
@@ -132,14 +135,16 @@ async def handle_event(self, event):
 
         split_output = output.split("\n")
         if " 4" in split_output:
+            description = f"Duplicate Host Header Tolerated"
             await self.emit_event(
                 {
                     "host": str(event.host),
-                    "url": event.data["url"],
-                    "description": f"Duplicate Host Header Tolerated",
+                    "url": url,
+                    "description": description,
                 },
                 "FINDING",
                 event,
+                context=f"{{module}} scanned {event.data['url']} and identified {{event.type}}: {description}",
             )
 
         # host header overrides
@@ -163,7 +168,7 @@ async def handle_event(self, event):
             override_headers[oh] = f"{subdomain_tag}.{self.domain}"
 
         output = await self.helpers.curl(
-            url=event.data["url"],
+            url=url,
             headers=override_headers,
             cookies=added_cookies,
         )
@@ -172,12 +177,14 @@ async def handle_event(self, event):
 
         # emit all the domain reflections we found
         for dr in domain_reflections:
+            description = f"Possible Host header injection. Injection technique: {dr}"
             await self.emit_event(
                 {
                     "host": str(event.host),
-                    "url": event.data["url"],
-                    "description": f"Possible Host header injection. Injection technique: {dr}",
+                    "url": url,
+                    "description": description,
                 },
                 "FINDING",
                 event,
+                context=f"{{module}} scanned {url} and identified {{event.type}}: {description}",
             )
diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py
index 7ea0c31c2..026e82f53 100644
--- a/bbot/modules/hunt.py
+++ b/bbot/modules/hunt.py
@@ -292,4 +292,9 @@ async def handle_event(self, event):
                     url = event.data.get("url", "")
                     if url:
                         data["url"] = url
-                    await self.emit_event(data, "FINDING", event)
+                    await self.emit_event(
+                        data,
+                        "FINDING",
+                        event,
+                        context=f'{{module}} analyzed HTTP_RESPONSE from {url} and identified {{event.type}}: potential {k.upper()} parameter "{p}"',
+                    )
diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py
index 2f4d22bad..7396df481 100644
--- a/bbot/modules/hunterio.py
+++ b/bbot/modules/hunterio.py
@@ -31,14 +31,27 @@ async def handle_event(self, event):
             if email:
                 email_event = self.make_event(email, "EMAIL_ADDRESS", event)
                 if email_event:
-                    await self.emit_event(email_event)
+                    await self.emit_event(
+                        email_event,
+                        context=f'{{module}} queried Hunter.IO API for "{query}" and found {{event.type}}: {{event.data}}',
+                    )
                     for source in sources:
                         domain = source.get("domain", "")
                         if domain:
-                            await self.emit_event(domain, "DNS_NAME", email_event)
+                            await self.emit_event(
+                                domain,
+                                "DNS_NAME",
+                                email_event,
+                                context=f"{{module}} originally found {email} at {{event.type}}: {{event.data}}",
+                            )
                         url = source.get("uri", "")
                         if url:
-                            await self.emit_event(url, "URL_UNVERIFIED", email_event)
+                            await self.emit_event(
+                                url,
+                                "URL_UNVERIFIED",
+                                email_event,
+                                context=f"{{module}} originally found {email} at {{event.type}}: {{event.data}}",
+                            )
 
     async def query(self, query):
         emails = []
diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index 682b79804..07b1e2943 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -234,6 +234,7 @@ class safety_counter_obj:
                 {"severity": "LOW", "host": str(event.host), "url": normalized_url, "description": description},
                 "VULNERABILITY",
                 event,
+                context=f"{{module}} detected low {{event.type}}: IIS shortname enumeration",
             )
             if not self.config.get("detect_only"):
                 for detection in detections:
@@ -323,7 +324,13 @@ class safety_counter_obj:
                             hint_type = "shortname-file"
                         else:
                             hint_type = "shortname-directory"
-                        await self.emit_event(f"{normalized_url}/{url_hint}", "URL_HINT", event, tags=[hint_type])
+                        await self.emit_event(
+                            f"{normalized_url}/{url_hint}",
+                            "URL_HINT",
+                            event,
+                            tags=[hint_type],
+                            context=f"{{module}} enumerated shortnames at {normalized_url} and found {{event.type}}: {url_hint}",
+                        )
 
     async def filter_event(self, event):
         if "dir" in event.tags:
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 310cee7d3..9ce36d60a 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -128,17 +128,22 @@ async def handle_event(self, event):
         # speculate URL_UNVERIFIED from URL or any event with "url" attribute
         event_is_url = event.type == "URL"
         event_has_url = isinstance(event.data, dict) and "url" in event.data
+        event_tags = ["httpx-safe"] if event.type("CODE_REPOSITORY", "SOCIAL") else []
         if event_is_url or event_has_url:
             if event_is_url:
                 url = event.data
             else:
                 url = event.data["url"]
+            # only emit the url if it's not already in the event's history
             if not any(e.type == "URL_UNVERIFIED" and e.data == url for e in event.get_parents()):
-                tags = None
                 if self.helpers.is_spider_danger(event.parent, url):
-                    tags = ["spider-danger"]
+                    event_tags.append("spider-danger")
                 await self.emit_event(
-                    url, "URL_UNVERIFIED", tags=tags, parent=event, context="speculated {event.type}: {event.data}"
+                    url,
+                    "URL_UNVERIFIED",
+                    tags=event_tags,
+                    parent=event,
+                    context="speculated {event.type}: {event.data}",
                 )
 
         # IP_ADDRESS / DNS_NAME --> OPEN_TCP_PORT
diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py
index 2e4649f60..69976ebaf 100644
--- a/bbot/modules/internetdb.py
+++ b/bbot/modules/internetdb.py
@@ -79,7 +79,7 @@ async def handle_event(self, event):
             return
         if data:
             if r.status_code == 200:
-                await self._parse_response(data=data, event=event)
+                await self._parse_response(data=data, event=event, ip=ip)
             elif r.status_code == 404:
                 detail = data.get("detail", "")
                 if detail:
@@ -89,25 +89,44 @@ async def handle_event(self, event):
                 err_msg = data.get("msg", "")
                 self.verbose(f"Shodan error for {ip}: {err_data}: {err_msg}")
 
-    async def _parse_response(self, data: dict, event):
+    async def _parse_response(self, data: dict, event, ip):
         """Handles emitting events from returned JSON"""
         data: dict  # has keys: cpes, hostnames, ip, ports, tags, vulns
+        ip = str(ip)
+        query_host = ip if event.data == ip else f"{event.data} ({ip})"
         # ip is a string, ports is a list of ports, the rest is a list of strings
         for hostname in data.get("hostnames", []):
-            await self.emit_event(hostname, "DNS_NAME", parent=event)
+            if hostname != event.data:
+                await self.emit_event(
+                    hostname,
+                    "DNS_NAME",
+                    parent=event,
+                    context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.data}}',
+                )
         for cpe in data.get("cpes", []):
-            await self.emit_event({"technology": cpe, "host": str(event.host)}, "TECHNOLOGY", parent=event)
+            await self.emit_event(
+                {"technology": cpe, "host": str(event.host)},
+                "TECHNOLOGY",
+                parent=event,
+                context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.data}}',
+            )
         for port in data.get("ports", []):
             await self.emit_event(
-                self.helpers.make_netloc(event.data, port), "OPEN_TCP_PORT", parent=event, internal=True, quick=True
+                self.helpers.make_netloc(event.data, port),
+                "OPEN_TCP_PORT",
+                parent=event,
+                internal=True,
+                quick=True,
+                context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found {{event.type}}: {{event.data}}',
             )
         vulns = data.get("vulns", [])
         if vulns:
             vulns_str = ", ".join([str(v) for v in vulns])
             await self.emit_event(
-                {"description": f"Shodan reported verified vulnerabilities: {vulns_str}", "host": str(event.host)},
+                {"description": f"Shodan reported possible vulnerabilities: {vulns_str}", "host": str(event.host)},
                 "FINDING",
                 parent=event,
+                context=f'{{module}} queried Shodan\'s InternetDB API for "{query_host}" and found potential {{event.type}}: {vulns_str}',
             )
 
     def get_ip(self, event):
diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py
index 6e2bf4a25..af7dd5d94 100644
--- a/bbot/modules/ip2location.py
+++ b/bbot/modules/ip2location.py
@@ -62,4 +62,15 @@ async def handle_event(self, event):
             if error_msg:
                 self.warning(error_msg)
         elif geo_data:
-            await self.emit_event(geo_data, "GEOLOCATION", event)
+            country = geo_data.get("country_name", "unknown country")
+            region = geo_data.get("region_name", "unknown region")
+            city = geo_data.get("city_name", "unknown city")
+            lat = geo_data.get("latitude", "")
+            long = geo_data.get("longitude", "")
+            description = f"{city}, {region}, {country} ({lat}, {long})"
+            await self.emit_event(
+                geo_data,
+                "GEOLOCATION",
+                event,
+                context=f'{{module}} queried IP2Location API for "{event.data}" and found {{event.type}}: {description}',
+            )
diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py
index e4cc1dd55..3aab345f2 100644
--- a/bbot/modules/ipneighbor.py
+++ b/bbot/modules/ipneighbor.py
@@ -39,4 +39,7 @@ async def handle_event(self, event):
                     if ip_event:
                         # keep the scope distance low to give it one more hop for DNS resolution
                         # ip_event.scope_distance = max(1, event.scope_distance)
-                        await self.emit_event(ip_event)
+                        await self.emit_event(
+                            ip_event,
+                            context="{module} produced {event.type}: {event.data}",
+                        )
diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py
index 0d81674a4..115a620ba 100644
--- a/bbot/modules/ipstack.py
+++ b/bbot/modules/ipstack.py
@@ -52,4 +52,15 @@ async def handle_event(self, event):
             if error_msg:
                 self.warning(error_msg)
         elif geo_data:
-            await self.emit_event(geo_data, "GEOLOCATION", event)
+            country = geo_data.get("country_name", "unknown country")
+            region = geo_data.get("region_name", "unknown region")
+            city = geo_data.get("city", "unknown city")
+            lat = geo_data.get("latitude", "")
+            long = geo_data.get("longitude", "")
+            description = f"{city}, {region}, {country} ({lat}, {long})"
+            await self.emit_event(
+                geo_data,
+                "GEOLOCATION",
+                event,
+                context=f'{{module}} queried ipstack.com\'s API for "{event.data}" and found {{event.type}}: {description}',
+            )
diff --git a/bbot/modules/newsletters.py b/bbot/modules/newsletters.py
index 11bb2a8b1..5f2bac729 100644
--- a/bbot/modules/newsletters.py
+++ b/bbot/modules/newsletters.py
@@ -52,4 +52,9 @@ async def handle_event(self, event):
                 if result:
                     description = f"Found a Newsletter Submission Form that could be used for email bombing attacks"
                     data = {"host": str(_event.host), "description": description, "url": _event.data["url"]}
-                    await self.emit_event(data, "FINDING", _event)
+                    await self.emit_event(
+                        data,
+                        "FINDING",
+                        _event,
+                        context="{module} searched HTTP_RESPONSE and identified {event.type}: a Newsletter Submission Form that could be used for email bombing attacks",
+                    )
diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py
index c5f9422e4..1c7d4cd89 100644
--- a/bbot/modules/ntlm.py
+++ b/bbot/modules/ntlm.py
@@ -86,9 +86,13 @@ async def setup(self):
 
     async def handle_event(self, event):
         found_hash = hash(f"{event.host}:{event.port}")
-        agen = self.handle_url(event)
+        if event.type == "URL":
+            url = event.data
+        else:
+            url = event.data["url"]
+        agen = self.handle_url(url, event)
         if found_hash not in self.found:
-            async for result, request_url in agen:
+            async for result, request_url, num_urls in agen:
                 if result and request_url:
                     self.found.add(found_hash)
                     await self.emit_event(
@@ -99,6 +103,7 @@ async def handle_event(self, event):
                         },
                         "FINDING",
                         parent=event,
+                        context=f"{{module}} tried {num_urls:,} NTLM endpoints against {url} and identified NTLM auth ({{event.type}}): {result}",
                     )
                     fqdn = result.get("FQDN", "")
                     if fqdn:
@@ -116,19 +121,13 @@ async def filter_event(self, event):
                     return True
         return False
 
-    async def handle_url(self, event):
-        if event.type == "URL":
-            urls = {
-                event.data,
-            }
-        else:
-            urls = {
-                event.data["url"],
-            }
+    async def handle_url(self, url, event):
+        urls = {url}
         if self.try_all:
             for endpoint in ntlm_discovery_endpoints:
                 urls.add(f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{endpoint}")
 
+        num_urls = len(urls)
         async for url, response in self.helpers.request_batch(
             urls, headers=NTLM_test_header, allow_redirects=False, timeout=self.http_timeout
         ):
@@ -138,6 +137,6 @@ async def handle_url(self, event):
                 try:
                     ntlm_resp_decoded = self.helpers.ntlm.ntlmdecode(ntlm_resp_b64)
                     if ntlm_resp_decoded:
-                        yield ntlm_resp_decoded, url
+                        yield ntlm_resp_decoded, url, num_urls
                 except NTLMError as e:
                     self.verbose(str(e))
diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py
index 1285df45c..d4dfcfb4e 100644
--- a/bbot/modules/oauth.py
+++ b/bbot/modules/oauth.py
@@ -71,24 +71,37 @@ async def handle_event(self, event):
                 )
                 if finding_event:
                     finding_event.source_domain = source_domain
-                    await self.emit_event(finding_event)
+                    await self.emit_event(
+                        finding_event,
+                        context=f'{{module}} identified {{event.type}}: OpenID Connect Endpoint for "{source_domain}" at {url}',
+                    )
                 url_event = self.make_event(
                     token_endpoint, "URL_UNVERIFIED", parent=event, tags=["affiliate", "oauth-token-endpoint"]
                 )
                 if url_event:
                     url_event.source_domain = source_domain
-                    await self.emit_event(url_event)
+                    await self.emit_event(
+                        url_event,
+                        context=f'{{module}} identified OpenID Connect Endpoint for "{source_domain}" at {{event.type}}: {url}',
+                    )
             for result in oidc_results:
                 if result not in (domain, event.data):
                     event_type = "URL_UNVERIFIED" if self.helpers.is_url(result) else "DNS_NAME"
-                    await self.emit_event(result, event_type, parent=event, tags=["affiliate"])
+                    await self.emit_event(
+                        result,
+                        event_type,
+                        parent=event,
+                        tags=["affiliate"],
+                        context=f'{{module}} analyzed OpenID configuration for "{source_domain}" and found {{event.type}}: {{event.data}}',
+                    )
 
         for oauth_task in oauth_tasks:
             url = await oauth_task
             if url:
+                description = f"Potentially Sprayable OAUTH Endpoint (domain: {source_domain}) at {url}"
                 oauth_finding = self.make_event(
                     {
-                        "description": f"Potentially Sprayable OAUTH Endpoint (domain: {source_domain}) at {url}",
+                        "description": description,
                         "host": event.host,
                         "url": url,
                     },
@@ -97,7 +110,10 @@ async def handle_event(self, event):
                 )
                 if oauth_finding:
                     oauth_finding.source_domain = source_domain
-                    await self.emit_event(oauth_finding)
+                    await self.emit_event(
+                        oauth_finding,
+                        context=f"{{module}} identified {{event.type}}: {description}",
+                    )
 
     def url_and_base(self, url):
         yield url
diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py
index 408167266..336ce002a 100644
--- a/bbot/modules/paramminer_headers.py
+++ b/bbot/modules/paramminer_headers.py
@@ -130,11 +130,13 @@ async def process_results(self, event, results):
             if reflection:
                 tags = ["http_reflection"]
             description = f"[Paramminer] {self.compare_mode.capitalize()}: [{result}] Reasons: [{reasons}] Reflection: [{str(reflection)}]"
+            reflected = "reflected " if reflection else ""
             await self.emit_event(
                 {"host": str(event.host), "url": url, "description": description},
                 "FINDING",
                 event,
                 tags=tags,
+                context=f'{{module}} scanned {url} and identified {{event.type}}: {reflected}{self.compare_mode} parameter: "{result}"',
             )
 
     async def handle_event(self, event):
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index 7c880aae8..53ddbb546 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -435,6 +435,23 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.bytes_to_human(459819198709) == "428.24GB"
     assert helpers.human_to_bytes("428.24GB") == 459819198709
 
+    # ordinals
+    assert integer_to_ordinal(1) == "1st"
+    assert integer_to_ordinal(2) == "2nd"
+    assert integer_to_ordinal(3) == "3rd"
+    assert integer_to_ordinal(4) == "4th"
+    assert integer_to_ordinal(11) == "11th"
+    assert integer_to_ordinal(12) == "12th"
+    assert integer_to_ordinal(13) == "13th"
+    assert integer_to_ordinal(21) == "21st"
+    assert integer_to_ordinal(22) == "22nd"
+    assert integer_to_ordinal(23) == "23rd"
+    assert integer_to_ordinal(101) == "101st"
+    assert integer_to_ordinal(111) == "111th"
+    assert integer_to_ordinal(112) == "112th"
+    assert integer_to_ordinal(113) == "113th"
+    assert integer_to_ordinal(0) == "0th"
+
     scan1 = bbot_scanner(modules="ipneighbor")
     await scan1.load_modules()
     assert int(helpers.get_size(scan1.modules["ipneighbor"])) > 0

From 9fa758003314c835c3a463ec397f3f3a37c9817a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 01:54:32 -0400
Subject: [PATCH 186/220] fix tests

---
 bbot/modules/base.py            | 6 ++++--
 bbot/modules/code_repository.py | 1 -
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index c0b0b399b..40b027c48 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -636,7 +636,8 @@ async def _worker(self):
                         else:
                             self.debug(f"Not accepting {event} because {reason}")
             except asyncio.CancelledError:
-                self.log.trace("Worker cancelled")
+                # this trace was used for debugging leaked CancelledErrors from inside httpx
+                # self.log.trace("Worker cancelled")
                 raise
         self.log.trace(f"Worker stopped")
 
@@ -1476,7 +1477,8 @@ async def _worker(self):
                     await self.forward_event(event, kwargs)
 
             except asyncio.CancelledError:
-                self.log.trace("Worker cancelled")
+                # this trace was used for debugging leaked CancelledErrors from inside httpx
+                # self.log.trace("Worker cancelled")
                 raise
             except BaseException as e:
                 self.critical(f"Critical failure in intercept module {self.name}: {e}")
diff --git a/bbot/modules/code_repository.py b/bbot/modules/code_repository.py
index f37c80407..ef76954a9 100644
--- a/bbot/modules/code_repository.py
+++ b/bbot/modules/code_repository.py
@@ -52,6 +52,5 @@ async def handle_event(self, event):
                     repo_event.scope_distance = event.scope_distance
                     await self.emit_event(
                         repo_event,
-                        context=f"detected {{event.type}}: {url}",
                         context=f"{{module}} detected {platform} {{event.type}} at {url}",
                     )

From d2a928098baad5a202232dd6783d3ce39c290d14 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 02:11:09 -0400
Subject: [PATCH 187/220] fix tests

---
 bbot/test/test_step_1/test_helpers.py | 30 +++++++++++++--------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index 53ddbb546..b149c5abd 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -436,21 +436,21 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver):
     assert helpers.human_to_bytes("428.24GB") == 459819198709
 
     # ordinals
-    assert integer_to_ordinal(1) == "1st"
-    assert integer_to_ordinal(2) == "2nd"
-    assert integer_to_ordinal(3) == "3rd"
-    assert integer_to_ordinal(4) == "4th"
-    assert integer_to_ordinal(11) == "11th"
-    assert integer_to_ordinal(12) == "12th"
-    assert integer_to_ordinal(13) == "13th"
-    assert integer_to_ordinal(21) == "21st"
-    assert integer_to_ordinal(22) == "22nd"
-    assert integer_to_ordinal(23) == "23rd"
-    assert integer_to_ordinal(101) == "101st"
-    assert integer_to_ordinal(111) == "111th"
-    assert integer_to_ordinal(112) == "112th"
-    assert integer_to_ordinal(113) == "113th"
-    assert integer_to_ordinal(0) == "0th"
+    assert helpers.integer_to_ordinal(1) == "1st"
+    assert helpers.integer_to_ordinal(2) == "2nd"
+    assert helpers.integer_to_ordinal(3) == "3rd"
+    assert helpers.integer_to_ordinal(4) == "4th"
+    assert helpers.integer_to_ordinal(11) == "11th"
+    assert helpers.integer_to_ordinal(12) == "12th"
+    assert helpers.integer_to_ordinal(13) == "13th"
+    assert helpers.integer_to_ordinal(21) == "21st"
+    assert helpers.integer_to_ordinal(22) == "22nd"
+    assert helpers.integer_to_ordinal(23) == "23rd"
+    assert helpers.integer_to_ordinal(101) == "101st"
+    assert helpers.integer_to_ordinal(111) == "111th"
+    assert helpers.integer_to_ordinal(112) == "112th"
+    assert helpers.integer_to_ordinal(113) == "113th"
+    assert helpers.integer_to_ordinal(0) == "0th"
 
     scan1 = bbot_scanner(modules="ipneighbor")
     await scan1.load_modules()

From 674d07438733de8d1fd2001849eb1b32adc5c3e9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 02:34:10 -0400
Subject: [PATCH 188/220] steady work on event context

---
 bbot/modules/internal/speculate.py |  2 +-
 bbot/modules/pgp.py                | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 157f4efc1..93278cc73 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -128,7 +128,7 @@ async def handle_event(self, event):
         # speculate URL_UNVERIFIED from URL or any event with "url" attribute
         event_is_url = event.type == "URL"
         event_has_url = isinstance(event.data, dict) and "url" in event.data
-        event_tags = ["httpx-safe"] if event.type("CODE_REPOSITORY", "SOCIAL") else []
+        event_tags = ["httpx-safe"] if event.type in ("CODE_REPOSITORY", "SOCIAL") else []
         if event_is_url or event_has_url:
             if event_is_url:
                 url = event.data
diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py
index 978ebcee0..b49033b0e 100644
--- a/bbot/modules/pgp.py
+++ b/bbot/modules/pgp.py
@@ -10,10 +10,13 @@ class pgp(subdomain_enum):
         "created_date": "2022-08-10",
         "author": "@TheTechromancer",
     }
+    # TODO: scan for Web Key Directory (/.well-known/openpgpkey/)
     options = {
         "search_urls": [
             "https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=<query>",
             "http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=<query>",
+            "https://pgpkeys.eu/pks/lookup?search=<query>&op=index",
+            "https://pgp.mit.edu/pks/lookup?search=<query>&op=index",
         ]
     }
     options_desc = {"search_urls": "PGP key servers to search"}
@@ -22,18 +25,20 @@ async def handle_event(self, event):
         query = self.make_query(event)
         results = await self.query(query)
         if results:
-            for hostname in results:
-                if not hostname == event:
-                    await self.emit_event(hostname, "EMAIL_ADDRESS", event, abort_if=self.abort_if)
+            for email, keyserver in results:
+                await self.emit_event(email, "EMAIL_ADDRESS", event, abort_if=self.abort_if,
+                context=f'{{module}} queried PGP keyserver {keyserver} for "{query}" and found {{event.type}}: {{event.data}}')
 
     async def query(self, query):
         results = set()
-        for url in self.config.get("search_urls", []):
-            url = url.replace("<query>", self.helpers.quote(query))
+        urls = self.config.get("search_urls", [])
+        urls = [url.replace("<query>", self.helpers.quote(query)) for url in urls]
+        async for url, response in self.helpers.request_batch(urls):
+            keyserver = self.helpers.urlparse(url).netloc
             response = await self.helpers.request(url)
             if response is not None:
                 for email in await self.helpers.re.extract_emails(response.text):
                     email = email.lower()
                     if email.endswith(query):
-                        results.add(email)
+                        results.add((email, keyserver))
         return results

From 46da056d92dfaa89e77a86c5a55bfcb7828ea4ef Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 02:48:35 -0400
Subject: [PATCH 189/220] steady work on event context

---
 bbot/modules/pgp.py         |  9 +++++++--
 bbot/modules/portscan.py    | 15 ++++++++++++---
 bbot/modules/robots.py      |  8 +++++++-
 bbot/modules/secretsdb.py   |  1 +
 bbot/modules/sitedossier.py |  8 +++++++-
 bbot/modules/skymem.py      | 16 ++++++++++++----
 bbot/modules/smuggler.py    |  1 +
 7 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py
index b49033b0e..0c53c2ad4 100644
--- a/bbot/modules/pgp.py
+++ b/bbot/modules/pgp.py
@@ -26,8 +26,13 @@ async def handle_event(self, event):
         results = await self.query(query)
         if results:
             for email, keyserver in results:
-                await self.emit_event(email, "EMAIL_ADDRESS", event, abort_if=self.abort_if,
-                context=f'{{module}} queried PGP keyserver {keyserver} for "{query}" and found {{event.type}}: {{event.data}}')
+                await self.emit_event(
+                    email,
+                    "EMAIL_ADDRESS",
+                    event,
+                    abort_if=self.abort_if,
+                    context=f'{{module}} queried PGP keyserver {keyserver} for "{query}" and found {{event.type}}: {{event.data}}',
+                )
 
     async def query(self, query):
         results = set()
diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index 6166af98b..bc6580492 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -83,8 +83,12 @@ async def handle_batch(self, *events):
             new_targets = []
             async for alive_host, _ in self.masscan(targets, ping=True):
                 source_event = self.scanned_tracker.search(alive_host)
-                if source_event.type != "DNS_NAME":
-                    await self.emit_event(alive_host, "IP_ADDRESS", source=source_event)
+                await self.emit_event(
+                    alive_host,
+                    "DNS_NAME",
+                    source=source_event,
+                    context=f"{{module}} pinged {source_event.data} and got a response: {{event.type}}: {{event.data}}",
+                )
                 new_targets.append(ipaddress.ip_network(alive_host, strict=False))
             targets = new_targets
 
@@ -95,7 +99,12 @@ async def handle_batch(self, *events):
                 if source_event.type == "DNS_NAME":
                     host = source_event.host
                 netloc = self.helpers.make_netloc(host, port)
-                await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event)
+                await self.emit_event(
+                    netloc,
+                    "OPEN_TCP_PORT",
+                    source=source_event,
+                    context=f"{{module}} executed a TCP SYN scan against {source_event.data} and found: {{event.type}}: {{event.data}}",
+                )
         else:
             self.verbose("Only ping sweep was requested, skipping TCP SYN scan")
 
diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py
index 7b2d7d262..ff71709b3 100644
--- a/bbot/modules/robots.py
+++ b/bbot/modules/robots.py
@@ -48,4 +48,10 @@ async def handle_event(self, event):
                         tags = []
                         if self.helpers.is_spider_danger(event, unverified_url):
                             tags.append("spider-danger")
-                        await self.emit_event(unverified_url, "URL_UNVERIFIED", parent=event, tags=tags)
+                        await self.emit_event(
+                            unverified_url,
+                            "URL_UNVERIFIED",
+                            parent=event,
+                            tags=tags,
+                            context=f"{{module}} found robots.txt at {url} and extracted {{event.type}}: {{event.data}}",
+                        )
diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py
index fcd92724a..2d70e538d 100644
--- a/bbot/modules/secretsdb.py
+++ b/bbot/modules/secretsdb.py
@@ -62,6 +62,7 @@ async def handle_event(self, event):
                 event_data,
                 "FINDING",
                 parent=event,
+                context=f"{{module}} searched HTTP response and found {{event.type}}: {description}",
             )
 
     def search_data(self, resp_body, resp_headers):
diff --git a/bbot/modules/sitedossier.py b/bbot/modules/sitedossier.py
index ec9cd717e..fe9015027 100644
--- a/bbot/modules/sitedossier.py
+++ b/bbot/modules/sitedossier.py
@@ -23,7 +23,13 @@ async def handle_event(self, event):
                 self.verbose(e)
                 continue
             if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
-                await self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if)
+                await self.emit_event(
+                    hostname,
+                    "DNS_NAME",
+                    event,
+                    abort_if=self.abort_if,
+                    context=f'{{module}} searched sitedossier.com for "{query}" and found {{event.type}}: {{event.data}}',
+                )
 
     async def query(self, query, parse_fn=None, request_fn=None):
         results = set()
diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py
index fe4bef77d..4d0409057 100644
--- a/bbot/modules/skymem.py
+++ b/bbot/modules/skymem.py
@@ -14,6 +14,7 @@ class skymem(emailformat):
     }
 
     base_url = "https://www.skymem.info"
+    _qsize = 1
 
     async def setup(self):
         self.next_page_regex = self.helpers.re.compile(r'<a href="/domain/([a-z0-9]+)\?p=', re.I)
@@ -26,8 +27,7 @@ async def handle_event(self, event):
         r = await self.request_with_fail_count(url)
         if not r:
             return
-        for email in await self.helpers.re.extract_emails(r.text):
-            await self.emit_event(email, "EMAIL_ADDRESS", parent=event)
+        responses = [r]
 
         # iterate through other pages
         domain_ids = await self.helpers.re.findall(self.next_page_regex, r.text)
@@ -38,11 +38,19 @@ async def handle_event(self, event):
             r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
             if not r2:
                 continue
-            for email in await self.helpers.re.extract_emails(r2.text):
-                await self.emit_event(email, "EMAIL_ADDRESS", parent=event)
+            responses.append(r2)
             pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text)
             if not pages:
                 break
             last_page = max([int(p) for p in pages])
             if page >= last_page:
                 break
+
+        for i, r in enumerate(responses):
+            for email in await self.helpers.re.extract_emails(r.text):
+                await self.emit_event(
+                    email,
+                    "EMAIL_ADDRESS",
+                    parent=event,
+                    context=f'{{module}} searched skymem.info for "{query}" and found {{event.type}} on page {i+1}: {{event.data}}',
+                )
diff --git a/bbot/modules/smuggler.py b/bbot/modules/smuggler.py
index 879059896..357fec188 100644
--- a/bbot/modules/smuggler.py
+++ b/bbot/modules/smuggler.py
@@ -43,4 +43,5 @@ async def handle_event(self, event):
                         {"host": str(event.host), "url": event.data, "description": description},
                         "FINDING",
                         parent=event,
+                        context=f"{{module}} scanned {event.data} and found HTTP smuggling ({{event.type}}): {text}",
                     )

From a3b646c07b3b226de46788b96d51118ed8adc379 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 02:59:08 -0400
Subject: [PATCH 190/220] whoops

---
 bbot/core/helpers/names_generator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py
index baa1f13db..c0a9ef4c3 100644
--- a/bbot/core/helpers/names_generator.py
+++ b/bbot/core/helpers/names_generator.py
@@ -190,8 +190,8 @@
     "rambunctious",
     "rapid_unscheduled",
     "raving",
-    "reductive",
     "reckless",
+    "reductive",
     "ripped",
     "sadistic",
     "satanic",

From 13da6de3f7b46d0dc8284c9f9c9f6d9878d58f4e Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 18:02:10 -0400
Subject: [PATCH 191/220] finished event context first pass

---
 bbot/modules/deadly/dastardly.py         |  2 ++
 bbot/modules/deadly/ffuf.py              |  8 +++++-
 bbot/modules/deadly/nuclei.py            |  3 +++
 bbot/modules/deadly/vhost.py             | 18 ++++++++++---
 bbot/modules/internal/cloud.py           |  4 ++-
 bbot/modules/internal/dns.py             |  6 +++--
 bbot/modules/internal/excavate.py        |  4 +--
 bbot/modules/output/asset_inventory.py   | 18 ++++++++++---
 bbot/modules/report/asn.py               | 20 +++++++++++---
 bbot/modules/social.py                   |  8 ++++--
 bbot/modules/sslcert.py                  |  6 ++++-
 bbot/modules/telerik.py                  | 11 ++++++--
 bbot/modules/templates/bucket.py         | 33 +++++++++++++++++++-----
 bbot/modules/templates/subdomain_enum.py | 12 ++++++++-
 bbot/modules/trufflehog.py               |  2 +-
 bbot/modules/url_manipulation.py         |  1 +
 bbot/modules/urlscan.py                  | 18 ++++++++++---
 bbot/modules/viewdns.py                  |  8 +++++-
 bbot/modules/wafw00f.py                  | 11 +++++++-
 bbot/modules/wappalyzer.py               |  6 ++++-
 bbot/modules/wayback.py                  | 10 ++++++-
 bbot/modules/zoomeye.py                  |  8 +++++-
 22 files changed, 177 insertions(+), 40 deletions(-)

diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py
index da0cc1ac8..4476b99ab 100644
--- a/bbot/modules/deadly/dastardly.py
+++ b/bbot/modules/deadly/dastardly.py
@@ -52,6 +52,7 @@ async def handle_event(self, event):
                             },
                             "FINDING",
                             event,
+                            context=f"{{module}} executed web scan against {host} and identified {{event.type}}: {failure.instance}",
                         )
                     else:
                         await self.emit_event(
@@ -63,6 +64,7 @@ async def handle_event(self, event):
                             },
                             "VULNERABILITY",
                             event,
+                            context=f"{{module}} executed web scan against {host} and identified {failure.severity.lower()} {{event.type}}: {failure.instance}",
                         )
 
     def construct_command(self, target):
diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py
index d761af763..ece21076a 100644
--- a/bbot/modules/deadly/ffuf.py
+++ b/bbot/modules/deadly/ffuf.py
@@ -71,7 +71,13 @@ async def handle_event(self, event):
 
         filters = await self.baseline_ffuf(fixed_url, exts=exts)
         async for r in self.execute_ffuf(self.tempfile, fixed_url, exts=exts, filters=filters):
-            await self.emit_event(r["url"], "URL_UNVERIFIED", parent=event, tags=[f"status-{r['status']}"])
+            await self.emit_event(
+                r["url"],
+                "URL_UNVERIFIED",
+                parent=event,
+                tags=[f"status-{r['status']}"],
+                context=f"{{module}} brute-forced {event.data} and found {{event.type}}: {{event.data}}",
+            )
 
     async def filter_event(self, event):
         if "endpoint" in event.tags:
diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py
index f49a64284..228a65898 100644
--- a/bbot/modules/deadly/nuclei.py
+++ b/bbot/modules/deadly/nuclei.py
@@ -157,6 +157,7 @@ async def handle_batch(self, *events):
                     {"technology": str(name).lower(), "url": url, "host": str(parent_event.host)},
                     "TECHNOLOGY",
                     parent_event,
+                    context=f"{{module}} scanned {url} and identified {{event.type}}: {str(name).lower()}",
                 )
                 continue
 
@@ -173,6 +174,7 @@ async def handle_batch(self, *events):
                     },
                     "FINDING",
                     parent_event,
+                    context=f"{{module}} scanned {url} and identified {{event.type}}: {description_string}",
                 )
             else:
                 await self.emit_event(
@@ -184,6 +186,7 @@ async def handle_batch(self, *events):
                     },
                     "VULNERABILITY",
                     parent_event,
+                    context=f"{{module}} scanned {url} and identified {severity.lower()} {{event.type}}: {description_string}",
                 )
 
     def correlate_event(self, events, host):
diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py
index f11106ae0..98991c53d 100644
--- a/bbot/modules/deadly/vhost.py
+++ b/bbot/modules/deadly/vhost.py
@@ -80,11 +80,23 @@ async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=F
             wordlist, host, exts=[""], suffix=basehost, filters=filters, mode="hostheader"
         ):
             found_vhost_b64 = r["input"]["FUZZ"]
-            vhost_dict = {"host": str(event.host), "url": host, "vhost": base64.b64decode(found_vhost_b64).decode()}
+            vhost_str = base64.b64decode(found_vhost_b64).decode()
+            vhost_dict = {"host": str(event.host), "url": host, "vhost": vhost_str}
             if f"{vhost_dict['vhost']}{basehost}" != event.parsed_url.netloc:
-                await self.emit_event(vhost_dict, "VHOST", parent=event)
+                await self.emit_event(
+                    vhost_dict,
+                    "VHOST",
+                    parent=event,
+                    context=f"{{module}} brute-forced virtual hosts for {event.data} and found {{event.type}}: {vhost_str}",
+                )
                 if skip_dns_host == False:
-                    await self.emit_event(f"{vhost_dict['vhost']}{basehost}", "DNS_NAME", parent=event, tags=["vhost"])
+                    await self.emit_event(
+                        f"{vhost_dict['vhost']}{basehost}",
+                        "DNS_NAME",
+                        parent=event,
+                        tags=["vhost"],
+                        context=f"{{module}} brute-forced virtual hosts for {event.data} and found {{event.type}}: {{event.data}}",
+                    )
 
                 yield vhost_dict["vhost"]
 
diff --git a/bbot/modules/internal/cloud.py b/bbot/modules/internal/cloud.py
index d17739175..6b2da62e4 100644
--- a/bbot/modules/internal/cloud.py
+++ b/bbot/modules/internal/cloud.py
@@ -67,9 +67,11 @@ async def handle_event(self, event, kwargs):
 
                             if event_type == "STORAGE_BUCKET":
                                 bucket_name, bucket_domain = match
+                                bucket_url = f"https://{bucket_name}.{bucket_domain}"
                                 _kwargs["data"] = {
                                     "name": bucket_name,
-                                    "url": f"https://{bucket_name}.{bucket_domain}",
+                                    "url": bucket_url,
+                                    "context": f"{{module}} analyzed {event.type} and found {{event.type}}: {bucket_url}",
                                 }
                                 await self.emit_event(**_kwargs)
 
diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py
index ccfe03496..276ebc50c 100644
--- a/bbot/modules/internal/dns.py
+++ b/bbot/modules/internal/dns.py
@@ -180,14 +180,16 @@ async def handle_event(self, event, kwargs):
                 "DNS_NAME",
                 module=self.host_module,
                 parent=event,
-                context="{event.parent.type} has host {event.type}: {event.host}",
             )
             # only emit the event if it's not already in the parent chain
             if parent_event is not None and (parent_event.always_emit or parent_event not in event.get_parents()):
                 parent_event.scope_distance = event.scope_distance
                 if "target" in event.tags:
                     parent_event.add_tag("target")
-                await self.emit_event(parent_event)
+                await self.emit_event(
+                    parent_event,
+                    context="{event.parent.type} has host {event.type}: {event.host}",
+                )
 
         # emit DNS children
         if emit_children:
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 9e41e5816..ed9751b1d 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -129,9 +129,7 @@ async def search(self, content, event, **kwargs):
                     url_event.add_tag("spider-danger")
 
                 self.excavate.debug(f"Found URL [{result}] from parsing [{event.data.get('url')}] with regex [{name}]")
-                await self.excavate.emit_event(
-                    url_event,
-                )
+                await self.excavate.emit_event(url_event)
                 if url_in_scope:
                     urls_found += 1
 
diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py
index 36d393c6b..a150c029d 100644
--- a/bbot/modules/output/asset_inventory.py
+++ b/bbot/modules/output/asset_inventory.py
@@ -185,23 +185,33 @@ async def finish(self):
                             host_event = self.make_event(
                                 asset.host, "DNS_NAME", parent=self.scan.root_event, raise_error=True
                             )
-                            await self.emit_event(host_event)
+                            await self.emit_event(
+                                host_event, context="{module} emitted previous result: {event.type}: {event.data}"
+                            )
                             for port in asset.ports:
                                 netloc = self.helpers.make_netloc(asset.host, port)
                                 open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", parent=host_event)
                                 if open_port_event:
-                                    await self.emit_event(open_port_event)
+                                    await self.emit_event(
+                                        open_port_event,
+                                        context="{module} emitted previous result: {event.type}: {event.data}",
+                                    )
                         else:
                             for ip in asset.ip_addresses:
                                 ip_event = self.make_event(
                                     ip, "IP_ADDRESS", parent=self.scan.root_event, raise_error=True
                                 )
-                                await self.emit_event(ip_event)
+                                await self.emit_event(
+                                    ip_event, context="{module} emitted previous result: {event.type}: {event.data}"
+                                )
                                 for port in asset.ports:
                                     netloc = self.helpers.make_netloc(ip, port)
                                     open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", parent=ip_event)
                                     if open_port_event:
-                                        await self.emit_event(open_port_event)
+                                        await self.emit_event(
+                                            open_port_event,
+                                            context="{module} emitted previous result: {event.type}: {event.data}",
+                                        )
             else:
                 self.warning(
                     f"use_previous=True was set but no previous asset inventory was found at {self.output_file}"
diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py
index 371212c66..43a03ec88 100644
--- a/bbot/modules/report/asn.py
+++ b/bbot/modules/report/asn.py
@@ -38,7 +38,7 @@ async def filter_event(self, event):
     async def handle_event(self, event):
         host = event.host
         if self.cache_get(host) == False:
-            asns = await self.get_asn(host)
+            asns, source = await self.get_asn(host)
             if not asns:
                 self.cache_put(self.unknown_asn)
             else:
@@ -46,11 +46,23 @@ async def handle_event(self, event):
                     emails = asn.pop("emails", [])
                     self.cache_put(asn)
                     asn_event = self.make_event(asn, "ASN", parent=event)
+                    asn_number = asn.get("asn", "")
+                    asn_desc = asn.get("description", "")
+                    asn_name = asn.get("name", "")
+                    asn_subnet = asn.get("subnet", "")
                     if not asn_event:
                         continue
-                    await self.emit_event(asn_event)
+                    await self.emit_event(
+                        asn_event,
+                        context=f"{{module}} checked {event.data} against {source} API and got {{event.type}}: AS{asn_number} ({asn_name}, {asn_desc}, {asn_subnet})",
+                    )
                     for email in emails:
-                        await self.emit_event(email, "EMAIL_ADDRESS", parent=asn_event)
+                        await self.emit_event(
+                            email,
+                            "EMAIL_ADDRESS",
+                            parent=asn_event,
+                            context=f"{{module}} retrieved details for AS{asn_number} and found {{event.type}}: {{event.data}}",
+                        )
 
     async def report(self):
         asn_data = sorted(self.asn_cache.items(), key=lambda x: self.asn_counts[x[0]], reverse=True)
@@ -104,7 +116,7 @@ async def get_asn(self, ip, retries=1):
                     self.sources.append(self.sources.pop(i))
                     self.verbose(f"Failed to contact {source}, retrying")
                     continue
-                return res
+                return res, source
         self.warning(f"Error retrieving ASN for {ip}")
         return []
 
diff --git a/bbot/modules/social.py b/bbot/modules/social.py
index 07adf3c7f..b80f6c18a 100644
--- a/bbot/modules/social.py
+++ b/bbot/modules/social.py
@@ -41,10 +41,14 @@ async def handle_event(self, event):
                 if not case_sensitive:
                     url = url.lower()
                     profile_name = profile_name.lower()
+                url = f"https://{url}"
                 social_event = self.make_event(
-                    {"platform": platform, "url": f"https://{url}", "profile_name": profile_name},
+                    {"platform": platform, "url": url, "profile_name": profile_name},
                     "SOCIAL",
                     parent=event,
                 )
                 social_event.scope_distance = event.scope_distance
-                await self.emit_event(social_event)
+                await self.emit_event(
+                    social_event,
+                    context=f"{{module}} detected {platform} {{event.type}} at {url}",
+                )
diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 71be0fd4d..7003117e7 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -87,7 +87,11 @@ async def handle_event(self, event):
                             else:
                                 tags = None
                             if ssl_event:
-                                await self.emit_event(ssl_event, tags=tags)
+                                await self.emit_event(
+                                    ssl_event,
+                                    tags=tags,
+                                    context=f"{{module}} parsed SSL certificate at {event.data} and found {{event.type}}: {{event.data}}",
+                                )
                         except ValidationError as e:
                             self.hugeinfo(f'Malformed {event_type} "{event_data}" at {event.data}')
                             self.debug(f"Invalid data at {host}:{port}: {e}")
diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index abe0893be..0bd68153e 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -218,6 +218,7 @@ async def handle_event(self, event):
                         {"host": str(event.host), "url": f"{event.data}{webresource}", "description": description},
                         "FINDING",
                         event,
+                        context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik RAU AXD Handler",
                     )
                     if self.config.get("exploit_RAU_crypto") == True:
                         hostname = urlparse(event.data).netloc
@@ -249,6 +250,7 @@ async def handle_event(self, event):
                                         },
                                         "VULNERABILITY",
                                         event,
+                                        context=f"{{module}} scanned {event.data} and identified critical {{event.type}}: description",
                                     )
                                     break
 
@@ -301,6 +303,7 @@ async def handle_event(self, event):
                         },
                         "FINDING",
                         event,
+                        context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik SpellCheckHandler",
                     )
 
             chartimagehandler = "ChartImage.axd?ImageName=bqYXJAqm315eEd6b%2bY4%2bGqZpe7a1kY0e89gfXli%2bjFw%3d"
@@ -319,30 +322,34 @@ async def handle_event(self, event):
                         },
                         "FINDING",
                         event,
+                        context=f"{{module}} scanned {event.data} and identified {{event.type}}: Telerik ChartImage AXD Handler",
                     )
 
         elif event.type == "HTTP_RESPONSE":
             resp_body = event.data.get("body", None)
+            url = event.data["url"]
             if resp_body:
                 if '":{"SerializedParameters":"' in resp_body:
                     await self.emit_event(
                         {
                             "host": str(event.host),
-                            "url": event.data["url"],
+                            "url": url,
                             "description": "Telerik DialogHandler [SerializedParameters] Detected in HTTP Response",
                         },
                         "FINDING",
                         event,
+                        context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik ChartImage AXD Handler",
                     )
                 elif '"_serializedConfiguration":"' in resp_body:
                     await self.emit_event(
                         {
                             "host": str(event.host),
-                            "url": event.data["url"],
+                            "url": url,
                             "description": "Telerik AsyncUpload [serializedConfiguration] Detected in HTTP Response",
                         },
                         "FINDING",
                         event,
+                        context=f"{{module}} searched HTTP_RESPONSE and identified {{event.type}}: Telerik AsyncUpload",
                     )
 
         # Check for RAD Controls in URL
diff --git a/bbot/modules/templates/bucket.py b/bbot/modules/templates/bucket.py
index 7b6a7ee61..9bfb1a3ae 100644
--- a/bbot/modules/templates/bucket.py
+++ b/bbot/modules/templates/bucket.py
@@ -52,8 +52,14 @@ async def handle_dns_name(self, event):
             for d in self.delimiters:
                 bucket_name = d.join(split)
                 buckets.add(bucket_name)
-        async for bucket_name, url, tags in self.brute_buckets(buckets, permutations=self.permutations):
-            await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", parent=event, tags=tags)
+        async for bucket_name, url, tags, num_buckets in self.brute_buckets(buckets, permutations=self.permutations):
+            await self.emit_event(
+                {"name": bucket_name, "url": url},
+                "STORAGE_BUCKET",
+                parent=event,
+                tags=tags,
+                context=f"{{module}} tried {num_buckets:,} bucket variations of {event.data} and found {{event.type}} at {url}",
+            )
 
     async def handle_storage_bucket(self, event):
         url = event.data["url"]
@@ -62,12 +68,24 @@ async def handle_storage_bucket(self, event):
             description, tags = await self._check_bucket_open(bucket_name, url)
             if description:
                 event_data = {"host": event.host, "url": url, "description": description}
-                await self.emit_event(event_data, "FINDING", parent=event, tags=tags)
-
-        async for bucket_name, url, tags in self.brute_buckets(
+                await self.emit_event(
+                    event_data,
+                    "FINDING",
+                    parent=event,
+                    tags=tags,
+                    context=f"{{module}} scanned {event.type} and identified {{event.type}}: {description}",
+                )
+
+        async for bucket_name, new_url, tags, num_buckets in self.brute_buckets(
             [bucket_name], permutations=self.permutations, omit_base=True
         ):
-            await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", parent=event, tags=tags)
+            await self.emit_event(
+                {"name": bucket_name, "url": new_url},
+                "STORAGE_BUCKET",
+                parent=event,
+                tags=tags,
+                context=f"{{module}} tried {num_buckets:,} variations of {url} and found {{event.type}} at {new_url}",
+            )
 
     async def brute_buckets(self, buckets, permutations=False, omit_base=False):
         buckets = set(buckets)
@@ -80,6 +98,7 @@ async def brute_buckets(self, buckets, permutations=False, omit_base=False):
         if omit_base:
             new_buckets = new_buckets - buckets
         new_buckets = [b for b in new_buckets if self.valid_bucket_name(b)]
+        num_buckets = len(new_buckets)
         tasks = []
         for base_domain in self.base_domains:
             for region in self.regions:
@@ -89,7 +108,7 @@ async def brute_buckets(self, buckets, permutations=False, omit_base=False):
         async for task in self.helpers.as_completed(tasks):
             existent_bucket, tags, bucket_name, url = await task
             if existent_bucket:
-                yield bucket_name, url, tags
+                yield bucket_name, url, tags, num_buckets
 
     async def _check_bucket_exists(self, bucket_name, url):
         self.debug(f'Checking if bucket exists: "{bucket_name}"')
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index c71a8e337..95c7995d3 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -30,6 +30,10 @@ class subdomain_enum(BaseModule):
     #   "lowest_parent": dedupe by lowest parent (lowest parent of www.api.test.evilcorp.com is api.test.evilcorp.com)
     dedup_strategy = "highest_parent"
 
+    @property
+    def source_pretty_name(self):
+        return f"{self.__class__.__name__} API"
+
     def _incoming_dedup_hash(self, event):
         """
         Determines the criteria for what is considered to be a duplicate event if `accept_dupes` is False.
@@ -48,7 +52,13 @@ async def handle_event(self, event):
                         self.verbose(e)
                         continue
                     if hostname and hostname.endswith(f".{query}") and not hostname == event.data:
-                        await self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if)
+                        await self.emit_event(
+                            hostname,
+                            "DNS_NAME",
+                            event,
+                            abort_if=self.abort_if,
+                            context=f'{{module}} searched {self.source_pretty_name} for "{query}" and found {{event.type}}: {{event.data}}',
+                        )
 
     async def request_url(self, query):
         url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}"
diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index d49480c1d..e32352d2d 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -78,7 +78,7 @@ async def handle_event(self, event):
                     data,
                     "FINDING",
                     event,
-                    context=f'{{module}} searched {event.type} using "{module}" method and found verified secret ({{event.type}}): {raw_result}',
+                    context=f'{{module}} searched {event.type} using "{module}" method and found possible secret ({{event.type}}): {raw_result}',
                 )
 
     async def execute_trufflehog(self, module, path):
diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py
index a77fdfe12..ef7cfe7f3 100644
--- a/bbot/modules/url_manipulation.py
+++ b/bbot/modules/url_manipulation.py
@@ -82,6 +82,7 @@ async def handle_event(self, event):
                                     {"description": description, "host": str(event.host), "url": event.data},
                                     "FINDING",
                                     parent=event,
+                                    context=f"{{module}} probed {event.data} and identified {{event.type}}: {description}",
                                 )
                         else:
                             self.debug(f"Status code changed to {str(subject_response.status_code)}, ignoring")
diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py
index 8230b7b3e..8b9b53bc8 100644
--- a/bbot/modules/urlscan.py
+++ b/bbot/modules/urlscan.py
@@ -27,17 +27,29 @@ async def handle_event(self, event):
                 domain_event = self.make_event(domain, "DNS_NAME", parent=event)
                 if domain_event:
                     if str(domain_event.host).endswith(query) and not str(domain_event.host) == str(event.host):
-                        await self.emit_event(domain_event, abort_if=self.abort_if)
+                        await self.emit_event(
+                            domain_event,
+                            abort_if=self.abort_if,
+                            context=f'{{module}} searched urlscan.io API for "{query}" and found {{event.type}}: {{event.data}}',
+                        )
                         parent_event = domain_event
             if url:
                 url_event = self.make_event(url, "URL_UNVERIFIED", parent=parent_event)
                 if url_event:
                     if str(url_event.host).endswith(query):
                         if self.urls:
-                            await self.emit_event(url_event, abort_if=self.abort_if)
+                            await self.emit_event(
+                                url_event,
+                                abort_if=self.abort_if,
+                                context=f'{{module}} searched urlscan.io API for "{query}" and found {{event.type}}: {{event.data}}',
+                            )
                         else:
                             await self.emit_event(
-                                str(url_event.host), "DNS_NAME", parent=event, abort_if=self.abort_if
+                                str(url_event.host),
+                                "DNS_NAME",
+                                parent=event,
+                                abort_if=self.abort_if,
+                                context=f'{{module}} searched urlscan.io API for "{query}" and found {{event.type}}: {{event.data}}',
                             )
                     else:
                         self.debug(f"{url_event.host} does not match {query}")
diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py
index 0bbdb33ea..96fd6fe94 100644
--- a/bbot/modules/viewdns.py
+++ b/bbot/modules/viewdns.py
@@ -28,7 +28,13 @@ async def setup(self):
     async def handle_event(self, event):
         _, query = self.helpers.split_domain(event.data)
         for domain, _ in await self.query(query):
-            await self.emit_event(domain, "DNS_NAME", parent=event, tags=["affiliate"])
+            await self.emit_event(
+                domain,
+                "DNS_NAME",
+                parent=event,
+                tags=["affiliate"],
+                context=f'{{module}} searched viewdns.info for "{query}" and found {{event.type}}: {{event.data}}',
+            )
 
     async def query(self, query):
         results = set()
diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py
index 711938db2..ac408765c 100644
--- a/bbot/modules/wafw00f.py
+++ b/bbot/modules/wafw00f.py
@@ -36,13 +36,21 @@ async def filter_event(self, event):
             return False, f"Invalid HTTP status code: {http_status}"
         return True, ""
 
+    def _incoming_dedup_hash(self, event):
+        return hash(f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/")
+
     async def handle_event(self, event):
         url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/"
         WW = await self.helpers.run_in_executor(wafw00f_main.WAFW00F, url, followredirect=False)
         waf_detections = await self.helpers.run_in_executor(WW.identwaf)
         if waf_detections:
             for waf in waf_detections:
-                await self.emit_event({"host": str(event.host), "url": url, "waf": waf}, "WAF", parent=event)
+                await self.emit_event(
+                    {"host": str(event.host), "url": url, "waf": waf},
+                    "WAF",
+                    parent=event,
+                    context=f"{{module}} scanned {url} and identified {{event.type}}: {waf}",
+                )
         else:
             if self.config.get("generic_detect") == True:
                 generic = await self.helpers.run_in_executor(WW.genericdetect)
@@ -56,4 +64,5 @@ async def handle_event(self, event):
                         },
                         "WAF",
                         parent=event,
+                        context=f"{{module}} scanned {url} and identified {{event.type}}: {waf}",
                     )
diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py
index 7fd8f5274..474b5c8d9 100644
--- a/bbot/modules/wappalyzer.py
+++ b/bbot/modules/wappalyzer.py
@@ -30,8 +30,12 @@ async def setup(self):
 
     async def handle_event(self, event):
         for res in await self.helpers.run_in_executor(self.wappalyze, event.data):
+            res = res.lower()
             await self.emit_event(
-                {"technology": res.lower(), "url": event.data["url"], "host": str(event.host)}, "TECHNOLOGY", event
+                {"technology": res, "url": event.data["url"], "host": str(event.host)},
+                "TECHNOLOGY",
+                event,
+                context=f"{{module}} analyzed HTTP_RESPONSE and identified {{event.type}}: {res}",
             )
 
     def wappalyze(self, data):
diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py
index 456c722a4..647ea342f 100644
--- a/bbot/modules/wayback.py
+++ b/bbot/modules/wayback.py
@@ -29,7 +29,13 @@ async def setup(self):
     async def handle_event(self, event):
         query = self.make_query(event)
         for result, event_type in await self.query(query):
-            await self.emit_event(result, event_type, event, abort_if=self.abort_if)
+            await self.emit_event(
+                result,
+                event_type,
+                event,
+                abort_if=self.abort_if,
+                context=f'{{module}} queried archive.org for "{query}" and found {{event.type}}: {{event.data}}',
+            )
 
     async def query(self, query):
         results = set()
@@ -58,6 +64,8 @@ async def query(self, query):
         dns_names = set()
         collapsed_urls = 0
         start_time = datetime.now()
+        # we consolidate URLs to cut down on garbage data
+        # this is CPU-intensive, so we do it in its own core.
         parsed_urls = await self.helpers.run_in_executor_mp(
             self.helpers.validators.collapse_urls,
             urls,
diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py
index 98397729a..fc4cfbfee 100644
--- a/bbot/modules/zoomeye.py
+++ b/bbot/modules/zoomeye.py
@@ -41,7 +41,13 @@ async def handle_event(self, event):
                 tags = []
                 if not hostname.endswith(f".{query}"):
                     tags = ["affiliate"]
-                await self.emit_event(hostname, "DNS_NAME", event, tags=tags)
+                await self.emit_event(
+                    hostname,
+                    "DNS_NAME",
+                    event,
+                    tags=tags,
+                    context=f'{{module}} searched ZoomEye API for "{query}" and found {{event.type}}: {{event.data}}',
+                )
 
     async def query(self, query):
         results = set()

From 0705113bf5030b3d4f83d51bb5beecab55069fd5 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 18:07:33 -0400
Subject: [PATCH 192/220] fix minor module bug

---
 bbot/modules/base.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 40b027c48..5cb4cc6fe 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -429,7 +429,8 @@ def make_event(self, *args, **kwargs):
         raise_error = kwargs.pop("raise_error", False)
         module = kwargs.pop("module", None)
         if module is None:
-            kwargs["module"] = self
+            if (not args) or getattr(args[0], "module", None) is None:
+                kwargs["module"] = self
         try:
             event = self.scan.make_event(*args, **kwargs)
         except ValidationError as e:

From 00ecfbd9cfa33dc25c956945b90639c1bdde125b Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 18:34:28 -0400
Subject: [PATCH 193/220] fix tests

---
 bbot/modules/portscan.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index bc6580492..d9bdc921a 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -82,12 +82,12 @@ async def handle_batch(self, *events):
         if self.ping_first or self.ping_only:
             new_targets = []
             async for alive_host, _ in self.masscan(targets, ping=True):
-                source_event = self.scanned_tracker.search(alive_host)
+                parent_event = self.scanned_tracker.search(alive_host)
                 await self.emit_event(
                     alive_host,
                     "DNS_NAME",
-                    source=source_event,
-                    context=f"{{module}} pinged {source_event.data} and got a response: {{event.type}}: {{event.data}}",
+                    source=parent_event,
+                    context=f"{{module}} pinged {parent_event.data} and got a response: {{event.type}}: {{event.data}}",
                 )
                 new_targets.append(ipaddress.ip_network(alive_host, strict=False))
             targets = new_targets
@@ -95,15 +95,15 @@ async def handle_batch(self, *events):
         # TCP SYN scan
         if not self.ping_only:
             async for host, port in self.masscan(targets):
-                source_event = self.scanned_tracker.search(host)
-                if source_event.type == "DNS_NAME":
-                    host = source_event.host
+                parent_event = self.scanned_tracker.search(host)
+                if parent_event.type == "DNS_NAME":
+                    host = parent_event.host
                 netloc = self.helpers.make_netloc(host, port)
                 await self.emit_event(
                     netloc,
                     "OPEN_TCP_PORT",
-                    source=source_event,
-                    context=f"{{module}} executed a TCP SYN scan against {source_event.data} and found: {{event.type}}: {{event.data}}",
+                    parent=parent_event,
+                    context=f"{{module}} executed a TCP SYN scan against {parent_event.data} and found: {{event.type}}: {{event.data}}",
                 )
         else:
             self.verbose("Only ping sweep was requested, skipping TCP SYN scan")

From d7aaf69866bc7858b415fd2971d8dbeead62dc94 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Sun, 26 May 2024 22:18:53 -0400
Subject: [PATCH 194/220] fix dotnetnuke test

---
 bbot/modules/dotnetnuke.py | 2 +-
 bbot/test/conftest.py      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index 9dfc48790..2207600e2 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -89,7 +89,7 @@ async def handle_event(self, event):
                         {"technology": "DotNetNuke", "url": event.data["url"], "host": str(event.host)},
                         "TECHNOLOGY",
                         event,
-                        context=f"{{module}} scanned {url} and found {{event.type}}: DotNetNuke",
+                        context=f"{{module}} scanned {event.data['url']} and found {{event.type}}: DotNetNuke",
                     )
                     detected = True
                     break
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 50b5c765e..b9732eb8f 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -119,9 +119,9 @@ def __init__(self, name):
         self.poll_task = None
 
     def mock_interaction(self, subdomain_tag, msg=None):
-        self.log.critical(f"Mocking interaction to subdomain tag: {subdomain_tag}")
+        self.log.info(f"Mocking interaction to subdomain tag: {subdomain_tag}")
         if msg is not None:
-            self.log.critical(msg)
+            self.log.info(msg)
         self.interactions.append(subdomain_tag)
 
     async def register(self, callback=None):

From 09194f31cb2df66011991f4a9e768422e1cb2390 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 04:31:36 +0000
Subject: [PATCH 195/220] Bump lxml from 5.2.1 to 5.2.2

Bumps [lxml](https://github.com/lxml/lxml) from 5.2.1 to 5.2.2.
- [Release notes](https://github.com/lxml/lxml/releases)
- [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt)
- [Commits](https://github.com/lxml/lxml/compare/lxml-5.2.1...lxml-5.2.2)

---
updated-dependencies:
- dependency-name: lxml
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 298 +++++++++++++++++++++++++---------------------------
 1 file changed, 143 insertions(+), 155 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index a1e43d13f..f84ab1a4a 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -870,165 +870,153 @@ files = [
 
 [[package]]
 name = "lxml"
-version = "5.2.1"
+version = "5.2.2"
 description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "lxml-5.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1f7785f4f789fdb522729ae465adcaa099e2a3441519df750ebdccc481d961a1"},
-    {file = "lxml-5.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6cc6ee342fb7fa2471bd9b6d6fdfc78925a697bf5c2bcd0a302e98b0d35bfad3"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:794f04eec78f1d0e35d9e0c36cbbb22e42d370dda1609fb03bcd7aeb458c6377"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c817d420c60a5183953c783b0547d9eb43b7b344a2c46f69513d5952a78cddf3"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2213afee476546a7f37c7a9b4ad4d74b1e112a6fafffc9185d6d21f043128c81"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b070bbe8d3f0f6147689bed981d19bbb33070225373338df755a46893528104a"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e02c5175f63effbd7c5e590399c118d5db6183bbfe8e0d118bdb5c2d1b48d937"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3dc773b2861b37b41a6136e0b72a1a44689a9c4c101e0cddb6b854016acc0aa8"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:d7520db34088c96cc0e0a3ad51a4fd5b401f279ee112aa2b7f8f976d8582606d"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:bcbf4af004f98793a95355980764b3d80d47117678118a44a80b721c9913436a"},
-    {file = "lxml-5.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a2b44bec7adf3e9305ce6cbfa47a4395667e744097faed97abb4728748ba7d47"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1c5bb205e9212d0ebddf946bc07e73fa245c864a5f90f341d11ce7b0b854475d"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2c9d147f754b1b0e723e6afb7ba1566ecb162fe4ea657f53d2139bbf894d050a"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:3545039fa4779be2df51d6395e91a810f57122290864918b172d5dc7ca5bb433"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a91481dbcddf1736c98a80b122afa0f7296eeb80b72344d7f45dc9f781551f56"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2ddfe41ddc81f29a4c44c8ce239eda5ade4e7fc305fb7311759dd6229a080052"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a7baf9ffc238e4bf401299f50e971a45bfcc10a785522541a6e3179c83eabf0a"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:31e9a882013c2f6bd2f2c974241bf4ba68c85eba943648ce88936d23209a2e01"},
-    {file = "lxml-5.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0a15438253b34e6362b2dc41475e7f80de76320f335e70c5528b7148cac253a1"},
-    {file = "lxml-5.2.1-cp310-cp310-win32.whl", hash = "sha256:6992030d43b916407c9aa52e9673612ff39a575523c5f4cf72cdef75365709a5"},
-    {file = "lxml-5.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:da052e7962ea2d5e5ef5bc0355d55007407087392cf465b7ad84ce5f3e25fe0f"},
-    {file = "lxml-5.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:70ac664a48aa64e5e635ae5566f5227f2ab7f66a3990d67566d9907edcbbf867"},
-    {file = "lxml-5.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1ae67b4e737cddc96c99461d2f75d218bdf7a0c3d3ad5604d1f5e7464a2f9ffe"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f18a5a84e16886898e51ab4b1d43acb3083c39b14c8caeb3589aabff0ee0b270"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6f2c8372b98208ce609c9e1d707f6918cc118fea4e2c754c9f0812c04ca116d"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:394ed3924d7a01b5bd9a0d9d946136e1c2f7b3dc337196d99e61740ed4bc6fe1"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d077bc40a1fe984e1a9931e801e42959a1e6598edc8a3223b061d30fbd26bbc"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:764b521b75701f60683500d8621841bec41a65eb739b8466000c6fdbc256c240"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:3a6b45da02336895da82b9d472cd274b22dc27a5cea1d4b793874eead23dd14f"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:5ea7b6766ac2dfe4bcac8b8595107665a18ef01f8c8343f00710b85096d1b53a"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:e196a4ff48310ba62e53a8e0f97ca2bca83cdd2fe2934d8b5cb0df0a841b193a"},
-    {file = "lxml-5.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:200e63525948e325d6a13a76ba2911f927ad399ef64f57898cf7c74e69b71095"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dae0ed02f6b075426accbf6b2863c3d0a7eacc1b41fb40f2251d931e50188dad"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:ab31a88a651039a07a3ae327d68ebdd8bc589b16938c09ef3f32a4b809dc96ef"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:df2e6f546c4df14bc81f9498bbc007fbb87669f1bb707c6138878c46b06f6510"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5dd1537e7cc06efd81371f5d1a992bd5ab156b2b4f88834ca852de4a8ea523fa"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9b9ec9c9978b708d488bec36b9e4c94d88fd12ccac3e62134a9d17ddba910ea9"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:8e77c69d5892cb5ba71703c4057091e31ccf534bd7f129307a4d084d90d014b8"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a8d5c70e04aac1eda5c829a26d1f75c6e5286c74743133d9f742cda8e53b9c2f"},
-    {file = "lxml-5.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c94e75445b00319c1fad60f3c98b09cd63fe1134a8a953dcd48989ef42318534"},
-    {file = "lxml-5.2.1-cp311-cp311-win32.whl", hash = "sha256:4951e4f7a5680a2db62f7f4ab2f84617674d36d2d76a729b9a8be4b59b3659be"},
-    {file = "lxml-5.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:5c670c0406bdc845b474b680b9a5456c561c65cf366f8db5a60154088c92d102"},
-    {file = "lxml-5.2.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:abc25c3cab9ec7fcd299b9bcb3b8d4a1231877e425c650fa1c7576c5107ab851"},
-    {file = "lxml-5.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6935bbf153f9a965f1e07c2649c0849d29832487c52bb4a5c5066031d8b44fd5"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d793bebb202a6000390a5390078e945bbb49855c29c7e4d56a85901326c3b5d9"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd5562927cdef7c4f5550374acbc117fd4ecc05b5007bdfa57cc5355864e0a4"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0e7259016bc4345a31af861fdce942b77c99049d6c2107ca07dc2bba2435c1d9"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:530e7c04f72002d2f334d5257c8a51bf409db0316feee7c87e4385043be136af"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59689a75ba8d7ffca577aefd017d08d659d86ad4585ccc73e43edbfc7476781a"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f9737bf36262046213a28e789cc82d82c6ef19c85a0cf05e75c670a33342ac2c"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:3a74c4f27167cb95c1d4af1c0b59e88b7f3e0182138db2501c353555f7ec57f4"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:68a2610dbe138fa8c5826b3f6d98a7cfc29707b850ddcc3e21910a6fe51f6ca0"},
-    {file = "lxml-5.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f0a1bc63a465b6d72569a9bba9f2ef0334c4e03958e043da1920299100bc7c08"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c2d35a1d047efd68027817b32ab1586c1169e60ca02c65d428ae815b593e65d4"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:79bd05260359170f78b181b59ce871673ed01ba048deef4bf49a36ab3e72e80b"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:865bad62df277c04beed9478fe665b9ef63eb28fe026d5dedcb89b537d2e2ea6"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:44f6c7caff88d988db017b9b0e4ab04934f11e3e72d478031efc7edcac6c622f"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:71e97313406ccf55d32cc98a533ee05c61e15d11b99215b237346171c179c0b0"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:057cdc6b86ab732cf361f8b4d8af87cf195a1f6dc5b0ff3de2dced242c2015e0"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f3bbbc998d42f8e561f347e798b85513ba4da324c2b3f9b7969e9c45b10f6169"},
-    {file = "lxml-5.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:491755202eb21a5e350dae00c6d9a17247769c64dcf62d8c788b5c135e179dc4"},
-    {file = "lxml-5.2.1-cp312-cp312-win32.whl", hash = "sha256:8de8f9d6caa7f25b204fc861718815d41cbcf27ee8f028c89c882a0cf4ae4134"},
-    {file = "lxml-5.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:f2a9efc53d5b714b8df2b4b3e992accf8ce5bbdfe544d74d5c6766c9e1146a3a"},
-    {file = "lxml-5.2.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:70a9768e1b9d79edca17890175ba915654ee1725975d69ab64813dd785a2bd5c"},
-    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0"},
-    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1"},
-    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863"},
-    {file = "lxml-5.2.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:5d5792e9b3fb8d16a19f46aa8208987cfeafe082363ee2745ea8b643d9cc5b45"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:88e22fc0a6684337d25c994381ed8a1580a6f5ebebd5ad41f89f663ff4ec2885"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:21c2e6b09565ba5b45ae161b438e033a86ad1736b8c838c766146eff8ceffff9"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_s390x.whl", hash = "sha256:afbbdb120d1e78d2ba8064a68058001b871154cc57787031b645c9142b937a62"},
-    {file = "lxml-5.2.1-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:627402ad8dea044dde2eccde4370560a2b750ef894c9578e1d4f8ffd54000461"},
-    {file = "lxml-5.2.1-cp36-cp36m-win32.whl", hash = "sha256:e89580a581bf478d8dcb97d9cd011d567768e8bc4095f8557b21c4d4c5fea7d0"},
-    {file = "lxml-5.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:59565f10607c244bc4c05c0c5fa0c190c990996e0c719d05deec7030c2aa8289"},
-    {file = "lxml-5.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:857500f88b17a6479202ff5fe5f580fc3404922cd02ab3716197adf1ef628029"},
-    {file = "lxml-5.2.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56c22432809085b3f3ae04e6e7bdd36883d7258fcd90e53ba7b2e463efc7a6af"},
-    {file = "lxml-5.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a55ee573116ba208932e2d1a037cc4b10d2c1cb264ced2184d00b18ce585b2c0"},
-    {file = "lxml-5.2.1-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:6cf58416653c5901e12624e4013708b6e11142956e7f35e7a83f1ab02f3fe456"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:64c2baa7774bc22dd4474248ba16fe1a7f611c13ac6123408694d4cc93d66dbd"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:74b28c6334cca4dd704e8004cba1955af0b778cf449142e581e404bd211fb619"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7221d49259aa1e5a8f00d3d28b1e0b76031655ca74bb287123ef56c3db92f213"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3dbe858ee582cbb2c6294dc85f55b5f19c918c2597855e950f34b660f1a5ede6"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:04ab5415bf6c86e0518d57240a96c4d1fcfc3cb370bb2ac2a732b67f579e5a04"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:6ab833e4735a7e5533711a6ea2df26459b96f9eec36d23f74cafe03631647c41"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f443cdef978430887ed55112b491f670bba6462cea7a7742ff8f14b7abb98d75"},
-    {file = "lxml-5.2.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8"},
-    {file = "lxml-5.2.1-cp37-cp37m-win32.whl", hash = "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd"},
-    {file = "lxml-5.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c"},
-    {file = "lxml-5.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b"},
-    {file = "lxml-5.2.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a"},
-    {file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585"},
-    {file = "lxml-5.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3d30321949861404323c50aebeb1943461a67cd51d4200ab02babc58bd06a86"},
-    {file = "lxml-5.2.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:b560e3aa4b1d49e0e6c847d72665384db35b2f5d45f8e6a5c0072e0283430533"},
-    {file = "lxml-5.2.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:058a1308914f20784c9f4674036527e7c04f7be6fb60f5d61353545aa7fcb739"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:adfb84ca6b87e06bc6b146dc7da7623395db1e31621c4785ad0658c5028b37d7"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:417d14450f06d51f363e41cace6488519038f940676ce9664b34ebf5653433a5"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a2dfe7e2473f9b59496247aad6e23b405ddf2e12ef0765677b0081c02d6c2c0b"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bf2e2458345d9bffb0d9ec16557d8858c9c88d2d11fed53998512504cd9df49b"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:58278b29cb89f3e43ff3e0c756abbd1518f3ee6adad9e35b51fb101c1c1daaec"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:64641a6068a16201366476731301441ce93457eb8452056f570133a6ceb15fca"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:78bfa756eab503673991bdcf464917ef7845a964903d3302c5f68417ecdc948c"},
-    {file = "lxml-5.2.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:11a04306fcba10cd9637e669fd73aa274c1c09ca64af79c041aa820ea992b637"},
-    {file = "lxml-5.2.1-cp38-cp38-win32.whl", hash = "sha256:66bc5eb8a323ed9894f8fa0ee6cb3e3fb2403d99aee635078fd19a8bc7a5a5da"},
-    {file = "lxml-5.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:9676bfc686fa6a3fa10cd4ae6b76cae8be26eb5ec6811d2a325636c460da1806"},
-    {file = "lxml-5.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cf22b41fdae514ee2f1691b6c3cdeae666d8b7fa9434de445f12bbeee0cf48dd"},
-    {file = "lxml-5.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec42088248c596dbd61d4ae8a5b004f97a4d91a9fd286f632e42e60b706718d7"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd53553ddad4a9c2f1f022756ae64abe16da1feb497edf4d9f87f99ec7cf86bd"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feaa45c0eae424d3e90d78823f3828e7dc42a42f21ed420db98da2c4ecf0a2cb"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddc678fb4c7e30cf830a2b5a8d869538bc55b28d6c68544d09c7d0d8f17694dc"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:853e074d4931dbcba7480d4dcab23d5c56bd9607f92825ab80ee2bd916edea53"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc4691d60512798304acb9207987e7b2b7c44627ea88b9d77489bbe3e6cc3bd4"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:beb72935a941965c52990f3a32d7f07ce869fe21c6af8b34bf6a277b33a345d3"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:6588c459c5627fefa30139be4d2e28a2c2a1d0d1c265aad2ba1935a7863a4913"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:588008b8497667f1ddca7c99f2f85ce8511f8f7871b4a06ceede68ab62dff64b"},
-    {file = "lxml-5.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:b6787b643356111dfd4032b5bffe26d2f8331556ecb79e15dacb9275da02866e"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7c17b64b0a6ef4e5affae6a3724010a7a66bda48a62cfe0674dabd46642e8b54"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:27aa20d45c2e0b8cd05da6d4759649170e8dfc4f4e5ef33a34d06f2d79075d57"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d4f2cc7060dc3646632d7f15fe68e2fa98f58e35dd5666cd525f3b35d3fed7f8"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff46d772d5f6f73564979cd77a4fffe55c916a05f3cb70e7c9c0590059fb29ef"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:96323338e6c14e958d775700ec8a88346014a85e5de73ac7967db0367582049b"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:52421b41ac99e9d91934e4d0d0fe7da9f02bfa7536bb4431b4c05c906c8c6919"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:7a7efd5b6d3e30d81ec68ab8a88252d7c7c6f13aaa875009fe3097eb4e30b84c"},
-    {file = "lxml-5.2.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ed777c1e8c99b63037b91f9d73a6aad20fd035d77ac84afcc205225f8f41188"},
-    {file = "lxml-5.2.1-cp39-cp39-win32.whl", hash = "sha256:644df54d729ef810dcd0f7732e50e5ad1bd0a135278ed8d6bcb06f33b6b6f708"},
-    {file = "lxml-5.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:9ca66b8e90daca431b7ca1408cae085d025326570e57749695d6a01454790e95"},
-    {file = "lxml-5.2.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9b0ff53900566bc6325ecde9181d89afadc59c5ffa39bddf084aaedfe3b06a11"},
-    {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd6037392f2d57793ab98d9e26798f44b8b4da2f2464388588f48ac52c489ea1"},
-    {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b9c07e7a45bb64e21df4b6aa623cb8ba214dfb47d2027d90eac197329bb5e94"},
-    {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3249cc2989d9090eeac5467e50e9ec2d40704fea9ab72f36b034ea34ee65ca98"},
-    {file = "lxml-5.2.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f42038016852ae51b4088b2862126535cc4fc85802bfe30dea3500fdfaf1864e"},
-    {file = "lxml-5.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:533658f8fbf056b70e434dff7e7aa611bcacb33e01f75de7f821810e48d1bb66"},
-    {file = "lxml-5.2.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:622020d4521e22fb371e15f580d153134bfb68d6a429d1342a25f051ec72df1c"},
-    {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efa7b51824aa0ee957ccd5a741c73e6851de55f40d807f08069eb4c5a26b2baa"},
-    {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c6ad0fbf105f6bcc9300c00010a2ffa44ea6f555df1a2ad95c88f5656104817"},
-    {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e233db59c8f76630c512ab4a4daf5a5986da5c3d5b44b8e9fc742f2a24dbd460"},
-    {file = "lxml-5.2.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6a014510830df1475176466b6087fc0c08b47a36714823e58d8b8d7709132a96"},
-    {file = "lxml-5.2.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:d38c8f50ecf57f0463399569aa388b232cf1a2ffb8f0a9a5412d0db57e054860"},
-    {file = "lxml-5.2.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5aea8212fb823e006b995c4dda533edcf98a893d941f173f6c9506126188860d"},
-    {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff097ae562e637409b429a7ac958a20aab237a0378c42dabaa1e3abf2f896e5f"},
-    {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f5d65c39f16717a47c36c756af0fb36144069c4718824b7533f803ecdf91138"},
-    {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:3d0c3dd24bb4605439bf91068598d00c6370684f8de4a67c2992683f6c309d6b"},
-    {file = "lxml-5.2.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e32be23d538753a8adb6c85bd539f5fd3b15cb987404327c569dfc5fd8366e85"},
-    {file = "lxml-5.2.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cc518cea79fd1e2f6c90baafa28906d4309d24f3a63e801d855e7424c5b34144"},
-    {file = "lxml-5.2.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a0af35bd8ebf84888373630f73f24e86bf016642fb8576fba49d3d6b560b7cbc"},
-    {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8aca2e3a72f37bfc7b14ba96d4056244001ddcc18382bd0daa087fd2e68a354"},
-    {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ca1e8188b26a819387b29c3895c47a5e618708fe6f787f3b1a471de2c4a94d9"},
-    {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c8ba129e6d3b0136a0f50345b2cb3db53f6bda5dd8c7f5d83fbccba97fb5dcb5"},
-    {file = "lxml-5.2.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e998e304036198b4f6914e6a1e2b6f925208a20e2042563d9734881150c6c246"},
-    {file = "lxml-5.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d3be9b2076112e51b323bdf6d5a7f8a798de55fb8d95fcb64bd179460cdc0704"},
-    {file = "lxml-5.2.1.tar.gz", hash = "sha256:3f7765e69bbce0906a7c74d5fe46d2c7a7596147318dbc08e4a2431f3060e306"},
+    {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:364d03207f3e603922d0d3932ef363d55bbf48e3647395765f9bfcbdf6d23632"},
+    {file = "lxml-5.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50127c186f191b8917ea2fb8b206fbebe87fd414a6084d15568c27d0a21d60db"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74e4f025ef3db1c6da4460dd27c118d8cd136d0391da4e387a15e48e5c975147"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:981a06a3076997adf7c743dcd0d7a0415582661e2517c7d961493572e909aa1d"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aef5474d913d3b05e613906ba4090433c515e13ea49c837aca18bde190853dff"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e275ea572389e41e8b039ac076a46cb87ee6b8542df3fff26f5baab43713bca"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5b65529bb2f21ac7861a0e94fdbf5dc0daab41497d18223b46ee8515e5ad297"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bcc98f911f10278d1daf14b87d65325851a1d29153caaf146877ec37031d5f36"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:b47633251727c8fe279f34025844b3b3a3e40cd1b198356d003aa146258d13a2"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:fbc9d316552f9ef7bba39f4edfad4a734d3d6f93341232a9dddadec4f15d425f"},
+    {file = "lxml-5.2.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:13e69be35391ce72712184f69000cda04fc89689429179bc4c0ae5f0b7a8c21b"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3b6a30a9ab040b3f545b697cb3adbf3696c05a3a68aad172e3fd7ca73ab3c835"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a233bb68625a85126ac9f1fc66d24337d6e8a0f9207b688eec2e7c880f012ec0"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:dfa7c241073d8f2b8e8dbc7803c434f57dbb83ae2a3d7892dd068d99e96efe2c"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a7aca7964ac4bb07680d5c9d63b9d7028cace3e2d43175cb50bba8c5ad33316"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ae4073a60ab98529ab8a72ebf429f2a8cc612619a8c04e08bed27450d52103c0"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ffb2be176fed4457e445fe540617f0252a72a8bc56208fd65a690fdb1f57660b"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e290d79a4107d7d794634ce3e985b9ae4f920380a813717adf61804904dc4393"},
+    {file = "lxml-5.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96e85aa09274955bb6bd483eaf5b12abadade01010478154b0ec70284c1b1526"},
+    {file = "lxml-5.2.2-cp310-cp310-win32.whl", hash = "sha256:f956196ef61369f1685d14dad80611488d8dc1ef00be57c0c5a03064005b0f30"},
+    {file = "lxml-5.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:875a3f90d7eb5c5d77e529080d95140eacb3c6d13ad5b616ee8095447b1d22e7"},
+    {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:45f9494613160d0405682f9eee781c7e6d1bf45f819654eb249f8f46a2c22545"},
+    {file = "lxml-5.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0b3f2df149efb242cee2ffdeb6674b7f30d23c9a7af26595099afaf46ef4e88"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d28cb356f119a437cc58a13f8135ab8a4c8ece18159eb9194b0d269ec4e28083"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:657a972f46bbefdbba2d4f14413c0d079f9ae243bd68193cb5061b9732fa54c1"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b74b9ea10063efb77a965a8d5f4182806fbf59ed068b3c3fd6f30d2ac7bee734"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07542787f86112d46d07d4f3c4e7c760282011b354d012dc4141cc12a68cef5f"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:303f540ad2dddd35b92415b74b900c749ec2010e703ab3bfd6660979d01fd4ed"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2eb2227ce1ff998faf0cd7fe85bbf086aa41dfc5af3b1d80867ecfe75fb68df3"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:1d8a701774dfc42a2f0b8ccdfe7dbc140500d1049e0632a611985d943fcf12df"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:56793b7a1a091a7c286b5f4aa1fe4ae5d1446fe742d00cdf2ffb1077865db10d"},
+    {file = "lxml-5.2.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:eb00b549b13bd6d884c863554566095bf6fa9c3cecb2e7b399c4bc7904cb33b5"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1a2569a1f15ae6c8c64108a2cd2b4a858fc1e13d25846be0666fc144715e32ab"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:8cf85a6e40ff1f37fe0f25719aadf443686b1ac7652593dc53c7ef9b8492b115"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d237ba6664b8e60fd90b8549a149a74fcc675272e0e95539a00522e4ca688b04"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0b3f5016e00ae7630a4b83d0868fca1e3d494c78a75b1c7252606a3a1c5fc2ad"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23441e2b5339bc54dc949e9e675fa35efe858108404ef9aa92f0456929ef6fe8"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2fb0ba3e8566548d6c8e7dd82a8229ff47bd8fb8c2da237607ac8e5a1b8312e5"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:79d1fb9252e7e2cfe4de6e9a6610c7cbb99b9708e2c3e29057f487de5a9eaefa"},
+    {file = "lxml-5.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6dcc3d17eac1df7859ae01202e9bb11ffa8c98949dcbeb1069c8b9a75917e01b"},
+    {file = "lxml-5.2.2-cp311-cp311-win32.whl", hash = "sha256:4c30a2f83677876465f44c018830f608fa3c6a8a466eb223535035fbc16f3438"},
+    {file = "lxml-5.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:49095a38eb333aaf44c06052fd2ec3b8f23e19747ca7ec6f6c954ffea6dbf7be"},
+    {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7429e7faa1a60cad26ae4227f4dd0459efde239e494c7312624ce228e04f6391"},
+    {file = "lxml-5.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:50ccb5d355961c0f12f6cf24b7187dbabd5433f29e15147a67995474f27d1776"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc911208b18842a3a57266d8e51fc3cfaccee90a5351b92079beed912a7914c2"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33ce9e786753743159799fdf8e92a5da351158c4bfb6f2db0bf31e7892a1feb5"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec87c44f619380878bd49ca109669c9f221d9ae6883a5bcb3616785fa8f94c97"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08ea0f606808354eb8f2dfaac095963cb25d9d28e27edcc375d7b30ab01abbf6"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75a9632f1d4f698b2e6e2e1ada40e71f369b15d69baddb8968dcc8e683839b18"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:74da9f97daec6928567b48c90ea2c82a106b2d500f397eeb8941e47d30b1ca85"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:0969e92af09c5687d769731e3f39ed62427cc72176cebb54b7a9d52cc4fa3b73"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:9164361769b6ca7769079f4d426a41df6164879f7f3568be9086e15baca61466"},
+    {file = "lxml-5.2.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d26a618ae1766279f2660aca0081b2220aca6bd1aa06b2cf73f07383faf48927"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab67ed772c584b7ef2379797bf14b82df9aa5f7438c5b9a09624dd834c1c1aaf"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:3d1e35572a56941b32c239774d7e9ad724074d37f90c7a7d499ab98761bd80cf"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:8268cbcd48c5375f46e000adb1390572c98879eb4f77910c6053d25cc3ac2c67"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e282aedd63c639c07c3857097fc0e236f984ceb4089a8b284da1c526491e3f3d"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfdc2bfe69e9adf0df4915949c22a25b39d175d599bf98e7ddf620a13678585"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4aefd911793b5d2d7a921233a54c90329bf3d4a6817dc465f12ffdfe4fc7b8fe"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8b8df03a9e995b6211dafa63b32f9d405881518ff1ddd775db4e7b98fb545e1c"},
+    {file = "lxml-5.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f11ae142f3a322d44513de1018b50f474f8f736bc3cd91d969f464b5bfef8836"},
+    {file = "lxml-5.2.2-cp312-cp312-win32.whl", hash = "sha256:16a8326e51fcdffc886294c1e70b11ddccec836516a343f9ed0f82aac043c24a"},
+    {file = "lxml-5.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:bbc4b80af581e18568ff07f6395c02114d05f4865c2812a1f02f2eaecf0bfd48"},
+    {file = "lxml-5.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e3d9d13603410b72787579769469af730c38f2f25505573a5888a94b62b920f8"},
+    {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38b67afb0a06b8575948641c1d6d68e41b83a3abeae2ca9eed2ac59892b36706"},
+    {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c689d0d5381f56de7bd6966a4541bff6e08bf8d3871bbd89a0c6ab18aa699573"},
+    {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:cf2a978c795b54c539f47964ec05e35c05bd045db5ca1e8366988c7f2fe6b3ce"},
+    {file = "lxml-5.2.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:739e36ef7412b2bd940f75b278749106e6d025e40027c0b94a17ef7968d55d56"},
+    {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d8bbcd21769594dbba9c37d3c819e2d5847656ca99c747ddb31ac1701d0c0ed9"},
+    {file = "lxml-5.2.2-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:2304d3c93f2258ccf2cf7a6ba8c761d76ef84948d87bf9664e14d203da2cd264"},
+    {file = "lxml-5.2.2-cp36-cp36m-win32.whl", hash = "sha256:02437fb7308386867c8b7b0e5bc4cd4b04548b1c5d089ffb8e7b31009b961dc3"},
+    {file = "lxml-5.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196"},
+    {file = "lxml-5.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b590b39ef90c6b22ec0be925b211298e810b4856909c8ca60d27ffbca6c12e6"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:c2faf60c583af0d135e853c86ac2735ce178f0e338a3c7f9ae8f622fd2eb788c"},
+    {file = "lxml-5.2.2-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905"},
+    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7ff762670cada8e05b32bf1e4dc50b140790909caa8303cfddc4d702b71ea184"},
+    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6"},
+    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:a6d2092797b388342c1bc932077ad232f914351932353e2e8706851c870bca1f"},
+    {file = "lxml-5.2.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61"},
+    {file = "lxml-5.2.2-cp37-cp37m-win32.whl", hash = "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f"},
+    {file = "lxml-5.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40"},
+    {file = "lxml-5.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7ed07b3062b055d7a7f9d6557a251cc655eed0b3152b76de619516621c56f5d3"},
+    {file = "lxml-5.2.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f60fdd125d85bf9c279ffb8e94c78c51b3b6a37711464e1f5f31078b45002421"},
+    {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a7e24cb69ee5f32e003f50e016d5fde438010c1022c96738b04fc2423e61706"},
+    {file = "lxml-5.2.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23cfafd56887eaed93d07bc4547abd5e09d837a002b791e9767765492a75883f"},
+    {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19b4e485cd07b7d83e3fe3b72132e7df70bfac22b14fe4bf7a23822c3a35bff5"},
+    {file = "lxml-5.2.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:7ce7ad8abebe737ad6143d9d3bf94b88b93365ea30a5b81f6877ec9c0dee0a48"},
+    {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e49b052b768bb74f58c7dda4e0bdf7b79d43a9204ca584ffe1fb48a6f3c84c66"},
+    {file = "lxml-5.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d14a0d029a4e176795cef99c056d58067c06195e0c7e2dbb293bf95c08f772a3"},
+    {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:be49ad33819d7dcc28a309b86d4ed98e1a65f3075c6acd3cd4fe32103235222b"},
+    {file = "lxml-5.2.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a6d17e0370d2516d5bb9062c7b4cb731cff921fc875644c3d751ad857ba9c5b1"},
+    {file = "lxml-5.2.2-cp38-cp38-win32.whl", hash = "sha256:5b8c041b6265e08eac8a724b74b655404070b636a8dd6d7a13c3adc07882ef30"},
+    {file = "lxml-5.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:f61efaf4bed1cc0860e567d2ecb2363974d414f7f1f124b1df368bbf183453a6"},
+    {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fb91819461b1b56d06fa4bcf86617fac795f6a99d12239fb0c68dbeba41a0a30"},
+    {file = "lxml-5.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d4ed0c7cbecde7194cd3228c044e86bf73e30a23505af852857c09c24e77ec5d"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54401c77a63cc7d6dc4b4e173bb484f28a5607f3df71484709fe037c92d4f0ed"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:625e3ef310e7fa3a761d48ca7ea1f9d8718a32b1542e727d584d82f4453d5eeb"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:519895c99c815a1a24a926d5b60627ce5ea48e9f639a5cd328bda0515ea0f10c"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c7079d5eb1c1315a858bbf180000757db8ad904a89476653232db835c3114001"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:343ab62e9ca78094f2306aefed67dcfad61c4683f87eee48ff2fd74902447726"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:cd9e78285da6c9ba2d5c769628f43ef66d96ac3085e59b10ad4f3707980710d3"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_ppc64le.whl", hash = "sha256:546cf886f6242dff9ec206331209db9c8e1643ae642dea5fdbecae2453cb50fd"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_s390x.whl", hash = "sha256:02f6a8eb6512fdc2fd4ca10a49c341c4e109aa6e9448cc4859af5b949622715a"},
+    {file = "lxml-5.2.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:339ee4a4704bc724757cd5dd9dc8cf4d00980f5d3e6e06d5847c1b594ace68ab"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0a028b61a2e357ace98b1615fc03f76eb517cc028993964fe08ad514b1e8892d"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:f90e552ecbad426eab352e7b2933091f2be77115bb16f09f78404861c8322981"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:d83e2d94b69bf31ead2fa45f0acdef0757fa0458a129734f59f67f3d2eb7ef32"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a02d3c48f9bb1e10c7788d92c0c7db6f2002d024ab6e74d6f45ae33e3d0288a3"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6d68ce8e7b2075390e8ac1e1d3a99e8b6372c694bbe612632606d1d546794207"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:453d037e09a5176d92ec0fd282e934ed26d806331a8b70ab431a81e2fbabf56d"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:3b019d4ee84b683342af793b56bb35034bd749e4cbdd3d33f7d1107790f8c472"},
+    {file = "lxml-5.2.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb3942960f0beb9f46e2a71a3aca220d1ca32feb5a398656be934320804c0df9"},
+    {file = "lxml-5.2.2-cp39-cp39-win32.whl", hash = "sha256:ac6540c9fff6e3813d29d0403ee7a81897f1d8ecc09a8ff84d2eea70ede1cdbf"},
+    {file = "lxml-5.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:610b5c77428a50269f38a534057444c249976433f40f53e3b47e68349cca1425"},
+    {file = "lxml-5.2.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b537bd04d7ccd7c6350cdaaaad911f6312cbd61e6e6045542f781c7f8b2e99d2"},
+    {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4820c02195d6dfb7b8508ff276752f6b2ff8b64ae5d13ebe02e7667e035000b9"},
+    {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a09f6184f17a80897172863a655467da2b11151ec98ba8d7af89f17bf63dae"},
+    {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:76acba4c66c47d27c8365e7c10b3d8016a7da83d3191d053a58382311a8bf4e1"},
+    {file = "lxml-5.2.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b128092c927eaf485928cec0c28f6b8bead277e28acf56800e972aa2c2abd7a2"},
+    {file = "lxml-5.2.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ae791f6bd43305aade8c0e22f816b34f3b72b6c820477aab4d18473a37e8090b"},
+    {file = "lxml-5.2.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a2f6a1bc2460e643785a2cde17293bd7a8f990884b822f7bca47bee0a82fc66b"},
+    {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e8d351ff44c1638cb6e980623d517abd9f580d2e53bfcd18d8941c052a5a009"},
+    {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bec4bd9133420c5c52d562469c754f27c5c9e36ee06abc169612c959bd7dbb07"},
+    {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:55ce6b6d803890bd3cc89975fca9de1dff39729b43b73cb15ddd933b8bc20484"},
+    {file = "lxml-5.2.2-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ab6a358d1286498d80fe67bd3d69fcbc7d1359b45b41e74c4a26964ca99c3f8"},
+    {file = "lxml-5.2.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:06668e39e1f3c065349c51ac27ae430719d7806c026fec462e5693b08b95696b"},
+    {file = "lxml-5.2.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9cd5323344d8ebb9fb5e96da5de5ad4ebab993bbf51674259dbe9d7a18049525"},
+    {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89feb82ca055af0fe797a2323ec9043b26bc371365847dbe83c7fd2e2f181c34"},
+    {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e481bba1e11ba585fb06db666bfc23dbe181dbafc7b25776156120bf12e0d5a6"},
+    {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d6c6ea6a11ca0ff9cd0390b885984ed31157c168565702959c25e2191674a14"},
+    {file = "lxml-5.2.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3d98de734abee23e61f6b8c2e08a88453ada7d6486dc7cdc82922a03968928db"},
+    {file = "lxml-5.2.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:69ab77a1373f1e7563e0fb5a29a8440367dec051da6c7405333699d07444f511"},
+    {file = "lxml-5.2.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:34e17913c431f5ae01d8658dbf792fdc457073dcdfbb31dc0cc6ab256e664a8d"},
+    {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05f8757b03208c3f50097761be2dea0aba02e94f0dc7023ed73a7bb14ff11eb0"},
+    {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a520b4f9974b0a0a6ed73c2154de57cdfd0c8800f4f15ab2b73238ffed0b36e"},
+    {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5e097646944b66207023bc3c634827de858aebc226d5d4d6d16f0b77566ea182"},
+    {file = "lxml-5.2.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b5e4ef22ff25bfd4ede5f8fb30f7b24446345f3e79d9b7455aef2836437bc38a"},
+    {file = "lxml-5.2.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff69a9a0b4b17d78170c73abe2ab12084bdf1691550c5629ad1fe7849433f324"},
+    {file = "lxml-5.2.2.tar.gz", hash = "sha256:bb2dc4898180bea79863d5487e5f9c7c34297414bad54bcd0f0852aee9cfdb87"},
 ]
 
 [package.extras]

From 40bb650a295f238f5b303525315eadab484fcbdb Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 04:33:05 +0000
Subject: [PATCH 196/220] Bump ansible-runner from 2.3.6 to 2.4.0

Bumps [ansible-runner](https://github.com/ansible/ansible-runner) from 2.3.6 to 2.4.0.
- [Release notes](https://github.com/ansible/ansible-runner/releases)
- [Commits](https://github.com/ansible/ansible-runner/compare/2.3.6...2.4.0)

---
updated-dependencies:
- dependency-name: ansible-runner
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index a1e43d13f..030a46588 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -46,13 +46,13 @@ resolvelib = ">=0.5.3,<1.1.0"
 
 [[package]]
 name = "ansible-runner"
-version = "2.3.6"
+version = "2.4.0"
 description = "\"Consistent Ansible Python API and CLI with container and process isolation runtime capabilities\""
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 files = [
-    {file = "ansible-runner-2.3.6.tar.gz", hash = "sha256:b2174a12dcad2dc2f342ea82876898f568a0b66c53568600bf80577158fcba1c"},
-    {file = "ansible_runner-2.3.6-py3-none-any.whl", hash = "sha256:4f153d9c3000a61b82d7253ca292849e3ad2c5d68dfff4377a6b98c4e6ff6c3e"},
+    {file = "ansible-runner-2.4.0.tar.gz", hash = "sha256:82d02b2548830f37a53517b65c823c4af371069406c7d213b5c9041d45e0c5b6"},
+    {file = "ansible_runner-2.4.0-py3-none-any.whl", hash = "sha256:a3f592ae4cdfa62a72ad15de60da9c8210f376d67f495c4a78d4cf1dc7ccdf89"},
 ]
 
 [package.dependencies]
@@ -61,7 +61,6 @@ packaging = "*"
 pexpect = ">=4.5"
 python-daemon = "*"
 pyyaml = "*"
-six = "*"
 
 [[package]]
 name = "antlr4-python3-runtime"

From e757cc6e7fcce058ce78aff9f8f8fc2407eb9ba0 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 27 May 2024 01:29:08 -0400
Subject: [PATCH 197/220] fix skymem

---
 bbot/modules/skymem.py | 27 +++++++++++++--------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py
index 4d0409057..4aac709b3 100644
--- a/bbot/modules/skymem.py
+++ b/bbot/modules/skymem.py
@@ -31,20 +31,19 @@ async def handle_event(self, event):
 
         # iterate through other pages
         domain_ids = await self.helpers.re.findall(self.next_page_regex, r.text)
-        if not domain_ids:
-            return
-        domain_id = domain_ids[0]
-        for page in range(2, 22):
-            r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
-            if not r2:
-                continue
-            responses.append(r2)
-            pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text)
-            if not pages:
-                break
-            last_page = max([int(p) for p in pages])
-            if page >= last_page:
-                break
+        if domain_ids:
+            domain_id = domain_ids[0]
+            for page in range(2, 22):
+                r2 = await self.request_with_fail_count(f"{self.base_url}/domain/{domain_id}?p={page}")
+                if not r2:
+                    continue
+                responses.append(r2)
+                pages = re.findall(r"/domain/" + domain_id + r"\?p=(\d+)", r2.text)
+                if not pages:
+                    break
+                last_page = max([int(p) for p in pages])
+                if page >= last_page:
+                    break
 
         for i, r in enumerate(responses):
             for email in await self.helpers.re.extract_emails(r.text):

From 950a7b22d87ab2e98c91ac0050968c8ef34af0a8 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 27 May 2024 02:27:27 -0400
Subject: [PATCH 198/220] fix masscan bug

---
 bbot/modules/portscan.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index d9bdc921a..6a971a569 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -83,6 +83,12 @@ async def handle_batch(self, *events):
             new_targets = []
             async for alive_host, _ in self.masscan(targets, ping=True):
                 parent_event = self.scanned_tracker.search(alive_host)
+                # masscan gets the occasional junk result
+                # this seems to be a side effect of it having its own TCP stack
+                # see https://github.com/robertdavidgraham/masscan/issues/397
+                if parent_event is None:
+                    self.debug(f"Failed to correlate {alive_host} to targets")
+                    continue
                 await self.emit_event(
                     alive_host,
                     "DNS_NAME",
@@ -96,6 +102,9 @@ async def handle_batch(self, *events):
         if not self.ping_only:
             async for host, port in self.masscan(targets):
                 parent_event = self.scanned_tracker.search(host)
+                if parent_event is None:
+                    self.debug(f"Failed to correlate {host} to targets")
+                    continue
                 if parent_event.type == "DNS_NAME":
                     host = parent_event.host
                 netloc = self.helpers.make_netloc(host, port)

From b49f282b796f9d781c9e27aa3cb87522bda16de5 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 27 May 2024 03:05:45 -0400
Subject: [PATCH 199/220] fix gowitness tests

---
 bbot/test/test_step_2/module_tests/test_module_gowitness.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_gowitness.py b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
index 3d309e47a..3c3ce5f0e 100644
--- a/bbot/test/test_step_2/module_tests/test_module_gowitness.py
+++ b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
@@ -29,10 +29,10 @@ async def setup_after_prep(self, module_test):
         # monkeypatch social
         old_emit_event = module_test.scan.modules["social"].emit_event
 
-        async def new_emit_event(event):
+        async def new_emit_event(event, **kwargs):
             if event.data["url"] == "https://github.com/blacklanternsecurity":
                 event.data["url"] = event.data["url"].replace("https://github.com", "http://127.0.0.1:8888")
-            await old_emit_event(event)
+            await old_emit_event(event, **kwargs)
 
         module_test.monkeypatch.setattr(module_test.scan.modules["social"], "emit_event", new_emit_event)
 

From 83014cfa87fd26e37cd4d944894b9471eb059127 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 18:31:42 +0000
Subject: [PATCH 200/220] Bump mkdocstrings from 0.24.3 to 0.25.1

Bumps [mkdocstrings](https://github.com/mkdocstrings/mkdocstrings) from 0.24.3 to 0.25.1.
- [Release notes](https://github.com/mkdocstrings/mkdocstrings/releases)
- [Changelog](https://github.com/mkdocstrings/mkdocstrings/blob/main/CHANGELOG.md)
- [Commits](https://github.com/mkdocstrings/mkdocstrings/compare/0.24.3...0.25.1)

---
updated-dependencies:
- dependency-name: mkdocstrings
  dependency-type: direct:development
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock    | 8 ++++----
 pyproject.toml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 8c8204c88..e3237ecbf 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1239,13 +1239,13 @@ files = [
 
 [[package]]
 name = "mkdocstrings"
-version = "0.24.3"
+version = "0.25.1"
 description = "Automatic documentation from sources, for MkDocs."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "mkdocstrings-0.24.3-py3-none-any.whl", hash = "sha256:5c9cf2a32958cd161d5428699b79c8b0988856b0d4a8c5baf8395fc1bf4087c3"},
-    {file = "mkdocstrings-0.24.3.tar.gz", hash = "sha256:f327b234eb8d2551a306735436e157d0a22d45f79963c60a8b585d5f7a94c1d2"},
+    {file = "mkdocstrings-0.25.1-py3-none-any.whl", hash = "sha256:da01fcc2670ad61888e8fe5b60afe9fee5781017d67431996832d63e887c2e51"},
+    {file = "mkdocstrings-0.25.1.tar.gz", hash = "sha256:c3a2515f31577f311a9ee58d089e4c51fc6046dbd9e9b4c3de4c3194667fe9bf"},
 ]
 
 [package.dependencies]
@@ -2507,4 +2507,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "f65140794e365b3fe93ff86e026591e528ab20ebcd278d4562a6cabe90ccac80"
+content-hash = "e4f9f370977b072794f49e3b9ac6fb383eb633b839d62693c02ef05494141b93"
diff --git a/pyproject.toml b/pyproject.toml
index 38fd76649..b66fcfb94 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,7 +70,7 @@ mkdocs = "^1.5.2"
 mkdocs-extra-sass-plugin = "^0.1.0"
 mkdocs-material = "^9.2.5"
 mkdocs-material-extensions = "^1.1.1"
-mkdocstrings = ">=0.22,<0.25"
+mkdocstrings = ">=0.22,<0.26"
 mkdocstrings-python = "^1.6.0"
 livereload = "^2.6.3"
 

From c48f2d9d64a71aa844f48a984fb9071c874181b4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 27 May 2024 19:30:24 +0000
Subject: [PATCH 201/220] Bump mkdocs-material from 9.5.18 to 9.5.25

Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.18 to 9.5.25.
- [Release notes](https://github.com/squidfunk/mkdocs-material/releases)
- [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG)
- [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.18...9.5.25)

---
updated-dependencies:
- dependency-name: mkdocs-material
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 poetry.lock | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index e3237ecbf..d4e7b049f 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1136,34 +1136,34 @@ files = [
 
 [[package]]
 name = "mkdocs"
-version = "1.5.3"
+version = "1.6.0"
 description = "Project documentation with Markdown."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "mkdocs-1.5.3-py3-none-any.whl", hash = "sha256:3b3a78e736b31158d64dbb2f8ba29bd46a379d0c6e324c2246c3bc3d2189cfc1"},
-    {file = "mkdocs-1.5.3.tar.gz", hash = "sha256:eb7c99214dcb945313ba30426c2451b735992c73c2e10838f76d09e39ff4d0e2"},
+    {file = "mkdocs-1.6.0-py3-none-any.whl", hash = "sha256:1eb5cb7676b7d89323e62b56235010216319217d4af5ddc543a91beb8d125ea7"},
+    {file = "mkdocs-1.6.0.tar.gz", hash = "sha256:a73f735824ef83a4f3bcb7a231dcab23f5a838f88b7efc54a0eef5fbdbc3c512"},
 ]
 
 [package.dependencies]
 click = ">=7.0"
 colorama = {version = ">=0.4", markers = "platform_system == \"Windows\""}
 ghp-import = ">=1.0"
-importlib-metadata = {version = ">=4.3", markers = "python_version < \"3.10\""}
+importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""}
 jinja2 = ">=2.11.1"
-markdown = ">=3.2.1"
+markdown = ">=3.3.6"
 markupsafe = ">=2.0.1"
 mergedeep = ">=1.3.4"
+mkdocs-get-deps = ">=0.2.0"
 packaging = ">=20.5"
 pathspec = ">=0.11.1"
-platformdirs = ">=2.2.0"
 pyyaml = ">=5.1"
 pyyaml-env-tag = ">=0.1"
 watchdog = ">=2.0"
 
 [package.extras]
 i18n = ["babel (>=2.9.0)"]
-min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.3)", "jinja2 (==2.11.1)", "markdown (==3.2.1)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "packaging (==20.5)", "pathspec (==0.11.1)", "platformdirs (==2.2.0)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "typing-extensions (==3.10)", "watchdog (==2.0)"]
+min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.4)", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"]
 
 [[package]]
 name = "mkdocs-autorefs"
@@ -1197,15 +1197,32 @@ beautifulsoup4 = ">=4.6.3"
 libsass = ">=0.15"
 mkdocs = ">=1.1"
 
+[[package]]
+name = "mkdocs-get-deps"
+version = "0.2.0"
+description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134"},
+    {file = "mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c"},
+]
+
+[package.dependencies]
+importlib-metadata = {version = ">=4.3", markers = "python_version < \"3.10\""}
+mergedeep = ">=1.3.4"
+platformdirs = ">=2.2.0"
+pyyaml = ">=5.1"
+
 [[package]]
 name = "mkdocs-material"
-version = "9.5.18"
+version = "9.5.25"
 description = "Documentation that simply works"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "mkdocs_material-9.5.18-py3-none-any.whl", hash = "sha256:1e0e27fc9fe239f9064318acf548771a4629d5fd5dfd45444fd80a953fe21eb4"},
-    {file = "mkdocs_material-9.5.18.tar.gz", hash = "sha256:a43f470947053fa2405c33995f282d24992c752a50114f23f30da9d8d0c57e62"},
+    {file = "mkdocs_material-9.5.25-py3-none-any.whl", hash = "sha256:68fdab047a0b9bfbefe79ce267e8a7daaf5128bcf7867065fcd201ee335fece1"},
+    {file = "mkdocs_material-9.5.25.tar.gz", hash = "sha256:d0662561efb725b712207e0ee01f035ca15633f29a64628e24f01ec99d7078f4"},
 ]
 
 [package.dependencies]
@@ -1213,7 +1230,7 @@ babel = ">=2.10,<3.0"
 colorama = ">=0.4,<1.0"
 jinja2 = ">=3.0,<4.0"
 markdown = ">=3.2,<4.0"
-mkdocs = ">=1.5.3,<1.6.0"
+mkdocs = ">=1.6,<2.0"
 mkdocs-material-extensions = ">=1.3,<2.0"
 paginate = ">=0.5,<1.0"
 pygments = ">=2.16,<3.0"

From f4d8419d6d10007670005f91bfd8aa20b3d6e450 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 27 May 2024 23:01:28 -0400
Subject: [PATCH 202/220] tests for json and csv

---
 bbot/core/event/base.py                       |  1 +
 bbot/modules/output/csv.py                    | 11 +++++++-
 bbot/test/test_step_2/module_tests/base.py    |  1 +
 .../module_tests/test_module_csv.py           |  6 ++++-
 .../module_tests/test_module_json.py          | 27 ++++++++++++++++---
 5 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ccbc8dda4..c715efcba 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1426,6 +1426,7 @@ def event_from_json(j, siem_friendly=False):
             "scans": j.get("scans", []),
             "tags": j.get("tags", []),
             "confidence": j.get("confidence", 5),
+            "context": j.get("discovery_context", None),
             "dummy": True,
         }
         if siem_friendly:
diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py
index 3fd28f545..d48e9cd1d 100644
--- a/bbot/modules/output/csv.py
+++ b/bbot/modules/output/csv.py
@@ -10,7 +10,15 @@ class CSV(BaseOutputModule):
     options = {"output_file": ""}
     options_desc = {"output_file": "Output to CSV file"}
 
-    header_row = ["Event type", "Event data", "IP Address", "Source Module", "Scope Distance", "Event Tags"]
+    header_row = [
+        "Event type",
+        "Event data",
+        "IP Address",
+        "Source Module",
+        "Scope Distance",
+        "Event Tags",
+        "Discovery Path",
+    ]
     filename = "output.csv"
     accept_dupes = False
 
@@ -56,6 +64,7 @@ async def handle_event(self, event):
                 "Source Module": str(getattr(event, "module_sequence", "")),
                 "Scope Distance": str(getattr(event, "scope_distance", "")),
                 "Event Tags": ",".join(sorted(list(getattr(event, "tags", [])))),
+                "Discovery Path": " --> ".join(getattr(event, "discovery_path", [])),
             }
         )
 
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index e8628bcf2..33e808212 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -118,6 +118,7 @@ async def module_test(
             self, httpx_mock, bbot_httpserver, bbot_httpserver_ssl, monkeypatch, request, caplog, capsys
         )
         module_test.log.info(f"Starting {self.name} module test")
+        await module_test.mock_dns({"blacklanternsecurity.com": {"A": ["127.0.0.88"]}})
         await self.setup_before_prep(module_test)
         await module_test.scan._prep()
         await self.setup_after_prep(module_test)
diff --git a/bbot/test/test_step_2/module_tests/test_module_csv.py b/bbot/test/test_step_2/module_tests/test_module_csv.py
index 6dc3f7cf1..5a9575372 100644
--- a/bbot/test/test_step_2/module_tests/test_module_csv.py
+++ b/bbot/test/test_step_2/module_tests/test_module_csv.py
@@ -7,5 +7,9 @@ async def setup_after_prep(self, module_test):
 
     def check(self, module_test, events):
         csv_file = module_test.scan.home / "output.csv"
+        context_data = f"Scan {module_test.scan.name} seeded with DNS_NAME: blacklanternsecurity.com"
+
         with open(csv_file) as f:
-            assert "blacklanternsecurity.com,127.0.0.5,TARGET" in f.read()
+            data = f.read()
+            assert "blacklanternsecurity.com,127.0.0.5,TARGET" in data
+            assert context_data in data
diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py
index 53affcd30..f320f4900 100644
--- a/bbot/test/test_step_2/module_tests/test_module_json.py
+++ b/bbot/test/test_step_2/module_tests/test_module_json.py
@@ -6,12 +6,33 @@
 
 class TestJSON(ModuleTestBase):
     def check(self, module_test, events):
+        scan_data = f"{module_test.scan.name} ({module_test.scan.id})"
+        dns_data = "blacklanternsecurity.com"
+        context_data = f"Scan {module_test.scan.name} seeded with DNS_NAME: blacklanternsecurity.com"
+
+        # json events
         txt_file = module_test.scan.home / "output.json"
         lines = list(module_test.scan.helpers.read_file(txt_file))
         assert lines
-        e = event_from_json(json.loads(lines[0]))
-        assert e.type == "SCAN"
-        assert e.data == f"{module_test.scan.name} ({module_test.scan.id})"
+        json_events = [json.loads(line) for line in lines]
+        scan_json = [e for e in json_events if e["type"] == "SCAN"]
+        dns_json = [e for e in json_events if e["type"] == "DNS_NAME"]
+        assert len(scan_json) == 1
+        assert len(dns_json) == 1
+        scan_json = scan_json[0]
+        dns_json = dns_json[0]
+        assert scan_json["data"] == scan_data
+        assert dns_json["data"] == dns_data
+        assert dns_json["discovery_context"] == context_data
+        assert dns_json["discovery_path"] == [context_data]
+
+        # event objects reconstructed from json
+        scan_reconstructed = event_from_json(scan_json)
+        dns_reconstructed = event_from_json(dns_json)
+        assert scan_reconstructed.data == scan_data
+        assert dns_reconstructed.data == dns_data
+        assert dns_reconstructed.discovery_context == context_data
+        assert dns_reconstructed.discovery_path == [context_data]
 
 
 class TestJSONSIEMFriendly(ModuleTestBase):

From 5a034ccda5f9b028385a668af5ee922253f57fe2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Mon, 27 May 2024 23:06:48 -0400
Subject: [PATCH 203/220] fix tests

---
 bbot/test/test_step_1/test_cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index fbe4324bc..b49365d85 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -112,7 +112,7 @@ async def test_cli_scan(monkeypatch):
 
     with open(scan_home / "output.csv") as f:
         lines = f.readlines()
-        assert lines[0] == "Event type,Event data,IP Address,Source Module,Scope Distance,Event Tags\n"
+        assert lines[0] == "Event type,Event data,IP Address,Source Module,Scope Distance,Event Tags,Discovery Path\n"
         assert len(lines) > 1, "output.csv is not long enough"
 
     ip_success = False

From 685e278f7ba4ca657aa65e67622a738db5a410c2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 01:30:24 -0400
Subject: [PATCH 204/220] fix github tests

---
 bbot/modules/dnsbrute_mutations.py                        | 5 -----
 bbot/test/test_step_1/test_events.py                      | 8 +++++++-
 .../test_step_2/module_tests/test_module_github_org.py    | 5 +++++
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py
index 562fc88d8..bb78114cf 100644
--- a/bbot/modules/dnsbrute_mutations.py
+++ b/bbot/modules/dnsbrute_mutations.py
@@ -111,11 +111,6 @@ def add_mutation(m):
                             parent_event = self.parent_events.get_host(hostname)
                             if parent_event is None:
                                 self.warning(f"Could not correlate parent event from: {hostname}")
-                                self.warning(self.parent_events._radix.dns_tree.root.children)
-                                self.warning(self.parent_events._radix.dns_tree.root.children["com"].children)
-                                self.warning(
-                                    self.parent_events._radix.dns_tree.root.children["com"].children["tesla"].children
-                                )
                                 parent_event = self.scan.root_event
                             mutation_run_ordinal = self.helpers.integer_to_ordinal(mutation_run)
                             await self.emit_event(
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index cd0165180..988054819 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -404,9 +404,11 @@ async def test_events(events, helpers):
     # test event serialization
     from bbot.core.event import event_from_json
 
-    db_event = scan.make_event("evilcorp.com:80", dummy=True)
+    db_event = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context")
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
+    assert db_event.discovery_context == "test context"
+    assert db_event.discovery_path == ["test context"]
     timestamp = db_event.timestamp.timestamp()
     json_event = db_event.json()
     assert json_event["scope_distance"] == 1
@@ -414,12 +416,16 @@ async def test_events(events, helpers):
     assert json_event["type"] == "OPEN_TCP_PORT"
     assert json_event["host"] == "evilcorp.com"
     assert json_event["timestamp"] == timestamp
+    assert json_event["discovery_context"] == "test context"
+    assert json_event["discovery_path"] == ["test context"]
     reconstituted_event = event_from_json(json_event)
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.timestamp() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
     assert reconstituted_event.type == "OPEN_TCP_PORT"
     assert reconstituted_event.host == "evilcorp.com"
+    assert reconstituted_event.discovery_context == "test context"
+    assert reconstituted_event.discovery_path == ["test context"]
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()
diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py
index 97107451f..d50c2ff09 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_org.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py
@@ -6,6 +6,11 @@ class TestGithub_Org(ModuleTestBase):
     modules_overrides = ["github_org", "speculate"]
 
     async def setup_before_prep(self, module_test):
+        await module_test.mock_dns({
+            "blacklanternsecurity.com": {"A": ["127.0.0.99"]},
+            "github.com": {"A": ["127.0.0.99"]}
+        })
+
         module_test.httpx_mock.add_response(url="https://api.github.com/zen")
         module_test.httpx_mock.add_response(
             url="https://api.github.com/orgs/blacklanternsecurity",

From 814d73e675bdb250b1844ce284a2fb26a2d9f952 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 01:30:36 -0400
Subject: [PATCH 205/220] blacked

---
 .../test_step_2/module_tests/test_module_github_org.py     | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_github_org.py b/bbot/test/test_step_2/module_tests/test_module_github_org.py
index d50c2ff09..ab403aba0 100644
--- a/bbot/test/test_step_2/module_tests/test_module_github_org.py
+++ b/bbot/test/test_step_2/module_tests/test_module_github_org.py
@@ -6,10 +6,9 @@ class TestGithub_Org(ModuleTestBase):
     modules_overrides = ["github_org", "speculate"]
 
     async def setup_before_prep(self, module_test):
-        await module_test.mock_dns({
-            "blacklanternsecurity.com": {"A": ["127.0.0.99"]},
-            "github.com": {"A": ["127.0.0.99"]}
-        })
+        await module_test.mock_dns(
+            {"blacklanternsecurity.com": {"A": ["127.0.0.99"]}, "github.com": {"A": ["127.0.0.99"]}}
+        )
 
         module_test.httpx_mock.add_response(url="https://api.github.com/zen")
         module_test.httpx_mock.add_response(

From 05c9a36cf702f77f7b136368ebe505542423325d Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 02:30:27 -0400
Subject: [PATCH 206/220] event context ux tweaks

---
 bbot/core/engine.py          | 8 ++++----
 bbot/modules/azure_tenant.py | 6 ++++--
 bbot/modules/gowitness.py    | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/bbot/core/engine.py b/bbot/core/engine.py
index a480e1c68..f9b601491 100644
--- a/bbot/core/engine.py
+++ b/bbot/core/engine.py
@@ -214,8 +214,8 @@ async def run_and_return(self, client_id, command_fn, *args, **kwargs):
             except BaseException as e:
                 error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
                 trace = traceback.format_exc()
-                self.log.error(error)
-                self.log.trace(trace)
+                self.log.debug(error)
+                self.log.debug(trace)
                 result = {"_e": (error, trace)}
             finally:
                 self.tasks.pop(client_id, None)
@@ -238,8 +238,8 @@ async def run_and_yield(self, client_id, command_fn, *args, **kwargs):
             except BaseException as e:
                 error = f"Error in {self.name}.{command_fn.__name__}({args}, {kwargs}): {e}"
                 trace = traceback.format_exc()
-                self.log.error(error)
-                self.log.trace(trace)
+                self.log.debug(error)
+                self.log.debug(trace)
                 result = {"_e": (error, trace)}
                 await self.send_socket_multipart(client_id, result)
             finally:
diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index 76b2e7f0a..7e4d136f6 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -51,14 +51,16 @@ async def handle_event(self, event):
                         if tenantname:
                             tenant_names.add(tenantname)
 
-            event_data = {"tenant-names": sorted(tenant_names), "domains": sorted(domains)}
+            tenant_names.sort()
+            event_data = {"tenant-names": tenant_names, "domains": sorted(domains)}
+            tenant_names_str = ",".join(tenant_names)
             if tenant_id is not None:
                 event_data["tenant-id"] = tenant_id
             await self.emit_event(
                 event_data,
                 "AZURE_TENANT",
                 parent=event,
-                context=f'{{module}} queried Outlook autodiscover for "{query}" and found {{event.type}}: {{event.data}}',
+                context=f'{{module}} queried Outlook autodiscover for "{query}" and found {{event.type}}: {tenant_names_str}',
             )
 
     async def query(self, domain):
diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py
index 59482ecbb..2d7c3679a 100644
--- a/bbot/modules/gowitness.py
+++ b/bbot/modules/gowitness.py
@@ -20,7 +20,7 @@ class gowitness(BaseModule):
         "resolution_x": 1440,
         "resolution_y": 900,
         "output_path": "",
-        "social": True,
+        "social": False,
         "idle_timeout": 1800,
     }
     options_desc = {

From 4c346fd0a2495fb47fbe3b906f3f652e08ec7702 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 02:37:11 -0400
Subject: [PATCH 207/220] fix git clone auth issue

---
 bbot/modules/git_clone.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py
index 8745d70d8..19d37f0b6 100644
--- a/bbot/modules/git_clone.py
+++ b/bbot/modules/git_clone.py
@@ -50,7 +50,7 @@ async def clone_git_repository(self, repository_url):
         else:
             url = repository_url
         command = ["git", "-C", self.output_dir, "clone", url]
-        output = await self.run_process(command)
+        output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"})
         if output.returncode == 0:
             folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
             return self.output_dir / folder_name

From 2c68b8dad108e36dc7dd91cc44481383e10be02f Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 02:47:59 -0400
Subject: [PATCH 208/220] fix regex warnings

---
 bbot/core/helpers/misc.py                     |   2 +-
 bbot/modules/git_clone.py                     |  15 +-
 .../module_tests/test_module_smuggler.py      |   4 +-
 .../module_tests/test_module_viewdns.py       | 182 ------------------
 4 files changed, 12 insertions(+), 191 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index b77aaa2b7..509c1b4e1 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -1454,7 +1454,7 @@ def search_dict_values(d, *regexes):
         ...         ]
         ...     }
         ... }
-        >>> url_regexes = re.compile(r'https?://[^\\s<>"]+|www\.[^\\s<>"]+')
+        >>> url_regexes = re.compile(r'https?://[^\\s<>"]+|www\\.[^\\s<>"]+')
         >>> list(search_dict_values(dict_to_search, url_regexes))
         ["https://www.evilcorp.com"]
     """
diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py
index 19d37f0b6..7942d498a 100644
--- a/bbot/modules/git_clone.py
+++ b/bbot/modules/git_clone.py
@@ -1,4 +1,5 @@
 from pathlib import Path
+from subprocess import CalledProcessError
 from bbot.modules.templates.github import github
 
 
@@ -50,9 +51,11 @@ async def clone_git_repository(self, repository_url):
         else:
             url = repository_url
         command = ["git", "-C", self.output_dir, "clone", url]
-        output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"})
-        if output.returncode == 0:
-            folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
-            return self.output_dir / folder_name
-        else:
-            return None
+        try:
+            output = await self.run_process(command, env={"GIT_TERMINAL_PROMPT": "0"}, check=True)
+        except CalledProcessError as e:
+            self.debug(f"Error cloning {url}. STDERR: {repr(e.stderr)}")
+            return
+
+        folder_name = output.stderr.split("Cloning into '")[1].split("'")[0]
+        return self.output_dir / folder_name
diff --git a/bbot/test/test_step_2/module_tests/test_module_smuggler.py b/bbot/test/test_step_2/module_tests/test_module_smuggler.py
index c887d684a..a40194b82 100644
--- a/bbot/test/test_step_2/module_tests/test_module_smuggler.py
+++ b/bbot/test/test_step_2/module_tests/test_module_smuggler.py
@@ -4,9 +4,9 @@
               ______ _              
      / _____)                       | |             
     ( (____  ____  _   _  ____  ____| | _____  ____ 
-     \____ \|    \| | | |/ _  |/ _  | || ___ |/ ___)
+     \\____ \|    \| | | |/ _  |/ _  | || ___ |/ ___)
      _____) ) | | | |_| ( (_| ( (_| | || ____| |    
-    (______/|_|_|_|____/ \___ |\___ |\_)_____)_|    
+    (______/|_|_|_|____/ \\___ |\\___ |\\_)_____)_|    
                         (_____(_____|               
 
          @defparam v1.1
diff --git a/bbot/test/test_step_2/module_tests/test_module_viewdns.py b/bbot/test/test_step_2/module_tests/test_module_viewdns.py
index 5a235b019..d196981ba 100644
--- a/bbot/test/test_step_2/module_tests/test_module_viewdns.py
+++ b/bbot/test/test_step_2/module_tests/test_module_viewdns.py
@@ -21,188 +21,6 @@ def check(self, module_test, events):
       <meta keywords="viewdns, dns, info, reverse ip, pagerank, portscan, port scan, lookup, records, whois, ipwhois, dnstools, web hosting, hosting, traceroute, dns report, dnsreport, ip location, ip location finder, spam, spam database, dnsbl, propagation, dns propagation checker, checker, china, chinese, firewall, great firewall, is my site down, is site down, site down, down, dns propagate">
       <meta description="This free tool will allow you to find domain names owned by an individual person or company.  Simply enter the email address or name of the person or company to find other domains registered using those same details.">
       <!-- form validation -->
-      <script language="JavaScript" type="text/javascript">
-         <!--
-         /* break us out of any containing iframes */
-         if (top != self) { top.location.replace(self.location.href); }
-         
-         function validate ( form )
-         {
-           for(v=0; v<form.elements.length; v++)
-           {
-             //trim element
-             if(!String.prototype.trim) {   
-               String.prototype.trim = function () {   
-                 return this.replace(/^\s+|\s+$/g,'');   
-               };   
-             } 
-             form.elements[v].value = form.elements[v].value.trim();
-             
-               
-               //check for null value
-               if (form.elements[v].value == "") 
-               {
-                 alert( "Value cannot be blank." );
-                 form.elements[v].focus();
-                 return false ;
-               }
-               //check if a domain parameter has a .
-               if (form.elements[v].name == "domain" || form.elements[v].name == "ip")
-               {
-                 if (form.elements[v].value.indexOf('.') == -1)
-                 {
-                     //no . in domain/ip
-                     alert("Invalid Domain/IP address.");
-                     form.elements[v].focus();
-                     return false;
-                 }
-               }
-               //ensure only whitelisted characters have been entered
-               if (form.elements[v].name == "asn")
-                 var whitelist = /^[0-9]$/;
-               else if (form.elements[v].name == "url")
-                 return true;
-               else
-                 var whitelist = /^[0-9a-zA-Z.\-\_]$/;
-               var data = form.elements[v].value;
-               for (var i=0;i<data.length;i++)
-               {
-                 var thisChar = data.charAt(i);
-                 if (!whitelist.test(thisChar))
-                 {
-                     alert( "Value contains invalid characters." );
-                     form.elements[v].focus();
-                     return false;
-                 }
-               }
-           }
-          
-           return true ;
-         }
-         //-->
-      </script>
-      <!-- end form validation -->
-      <!-- form validation -->
-      <script language="JavaScript" type="text/javascript">
-         <!--
-         function validateReg ( form )
-         {
-           for(v=0; v<form.elements.length; v++)
-           {
-               //check for null value
-               if (form.elements[v].name != "site")
-               {
-                   if (form.elements[v].value == "") 
-                   {
-                     alert( "Value cannot be blank." );
-                     form.elements[v].focus();
-                     return false ;
-                   }
-                 }
-                 //make sure password length 8 and passwords match
-                 if (document.register.pass1.value != document.register.pass2.value)
-                 {
-                     alert("Passwords do not match.");
-                     return false;
-                 }
-                 if (document.register.pass1.value.length < 8)
-                 {
-                     alert("Password must be at least 8 characters.");
-                     return false;
-                 }       
-                 
-                 //check email address is valid
-                 if (form.elements[v].name == "email")
-                 {
-                     var re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
-                 if (!re.test(form.elements[v].value))
-                 {
-                     alert("Email address is invalid.");
-                     return false;
-                 }
-             }
-           }
-          
-           return true ;
-         }
-         //-->
-      </script>
-      <!-- end form validation -->
-      <style type="text/css">
-         <!--
-            .mainlinks { text-decoration: none; color: #000000 }
-            -->
-      </style>
-      <!-- google analytics -->
-      <script type="text/javascript">
-         var _gaq = _gaq || [];
-         _gaq.push(['_setAccount', 'UA-20325459-1']);
-         _gaq.push(['_setDomainName', '.viewdns.info']);
-         _gaq.push(['_trackPageview']);
-         
-         (function() {
-           var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
-           ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
-           var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
-         })();
-         gtag('config', 'AW-1024044984');
-      </script>
-      <!-- /google analytics -->
-      <style>
-         #header ul {
-         list-style: none;
-         padding:0;
-         padding-bottom: 2px; 
-         margin:0;
-         }
-         #header li {
-         float: left;
-         border: 1px solid #CCCCCC;
-         border-bottom-width: 0;
-         margin: 0 0.5em 0 0;
-         }
-         #header li a {
-         padding: 0 1em;
-         color: #000000;
-         text-decoration: none;
-         }
-         #header #selected {
-         position: relative;
-         top: 2px;
-         padding-top: 5px;
-         padding-bottom:4px;
-         background: white;
-         font-weight: bold;
-         }
-         #header #notselected {
-         position: relative;
-         top: 5px;
-         padding-top: 2px;
-         padding-bottom: 3px;
-         background: #e1eae2;
-         }
-         body {
-         font-family: Verdana, sans-serif;
-         }
-      </style>
-      <link rel="apple-touch-icon" href="/apple-touch-icon.png"/>
-      <!-- FBPX -->
-      <script>
-         !function(f,b,e,v,n,t,s)
-         {if(f.fbq)return;n=f.fbq=function(){n.callMethod?
-         n.callMethod.apply(n,arguments):n.queue.push(arguments)};
-         if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
-         n.queue=[];t=b.createElement(e);t.async=!0;
-         t.src=v;s=b.getElementsByTagName(e)[0];
-         s.parentNode.insertBefore(t,s)}(window, document,'script',
-         'https://connect.facebook.net/en_US/fbevents.js');
-         fbq('init', '219605381956214');
-         fbq('track', 'PageView');
-      </script>
-      <noscript><img height="1" width="1" style="display:none"
-         src="https://www.facebook.com/tr?id=219605381956214&ev=PageView&noscript=1"
-         /></noscript>
-      <!-- FBPX -->
    </head>
    <body bgcolor="#ededed">
       <font face="Verdana">

From 1247d711c1401098f76a0f49ed70590564b50c57 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 02:53:32 -0400
Subject: [PATCH 209/220] fix azure_tenant tests

---
 bbot/modules/azure_tenant.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py
index 7e4d136f6..bd4a9b3dc 100644
--- a/bbot/modules/azure_tenant.py
+++ b/bbot/modules/azure_tenant.py
@@ -51,7 +51,7 @@ async def handle_event(self, event):
                         if tenantname:
                             tenant_names.add(tenantname)
 
-            tenant_names.sort()
+            tenant_names = sorted(tenant_names)
             event_data = {"tenant-names": tenant_names, "domains": sorted(domains)}
             tenant_names_str = ",".join(tenant_names)
             if tenant_id is not None:

From be814ebdde3bdbf6a8f92210ee17c78f0c2665ba Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 09:10:46 -0400
Subject: [PATCH 210/220] fix gowitness tests

---
 .../module_tests/test_module_gowitness.py     | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_gowitness.py b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
index 3c3ce5f0e..7dd72b41a 100644
--- a/bbot/test/test_step_2/module_tests/test_module_gowitness.py
+++ b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
@@ -40,8 +40,8 @@ def check(self, module_test, events):
         screenshots_path = self.home_dir / "scans" / module_test.scan.name / "gowitness" / "screenshots"
         screenshots = list(screenshots_path.glob("*.png"))
         assert (
-            len(screenshots) == 2
-        ), f"{len(screenshots):,} .png files found at {screenshots_path}, should have been 2"
+            len(screenshots) == 1
+        ), f"{len(screenshots):,} .png files found at {screenshots_path}, should have been 1"
         assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/"])
         assert 1 == len(
             [e for e in events if e.type == "URL_UNVERIFIED" and e.data == "https://fonts.googleapis.com/"]
@@ -50,8 +50,22 @@ def check(self, module_test, events):
         assert 1 == len(
             [e for e in events if e.type == "SOCIAL" and e.data["url"] == "http://127.0.0.1:8888/blacklanternsecurity"]
         )
-        assert 2 == len([e for e in events if e.type == "WEBSCREENSHOT"])
+        assert 1 == len([e for e in events if e.type == "WEBSCREENSHOT"])
         assert 1 == len([e for e in events if e.type == "WEBSCREENSHOT" and e.data["url"] == "http://127.0.0.1:8888/"])
+        assert len([e for e in events if e.type == "TECHNOLOGY"])
+
+
+class TestGowitness_Social(TestGowitness):
+    config_overrides = dict(TestGowitness.config_overrides)
+    config_overrides.update({"modules": {"gowitness": {"social": True}}})
+
+    def check(self, module_test, events):
+        screenshots_path = self.home_dir / "scans" / module_test.scan.name / "gowitness" / "screenshots"
+        screenshots = list(screenshots_path.glob("*.png"))
+        assert (
+            len(screenshots) == 2
+        ), f"{len(screenshots):,} .png files found at {screenshots_path}, should have been 2"
+        assert 2 == len([e for e in events if e.type == "WEBSCREENSHOT"])
         assert 1 == len(
             [
                 e
@@ -59,7 +73,6 @@ def check(self, module_test, events):
                 if e.type == "WEBSCREENSHOT" and e.data["url"] == "http://127.0.0.1:8888/blacklanternsecurity"
             ]
         )
-        assert len([e for e in events if e.type == "TECHNOLOGY"])
         assert 1 == len(
             [
                 e

From cd91ec119298ba7fc69e4916c27e3ff06c736022 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 13:45:12 -0400
Subject: [PATCH 211/220] fix telerik context

---
 bbot/modules/telerik.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py
index 0bd68153e..a64ef3c6f 100644
--- a/bbot/modules/telerik.py
+++ b/bbot/modules/telerik.py
@@ -250,7 +250,7 @@ async def handle_event(self, event):
                                         },
                                         "VULNERABILITY",
                                         event,
-                                        context=f"{{module}} scanned {event.data} and identified critical {{event.type}}: description",
+                                        context=f"{{module}} scanned {event.data} and identified critical {{event.type}}: {description}",
                                     )
                                     break
 

From 80444552ee9b7354c386f2d281089775c7e165a2 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 19:55:56 -0400
Subject: [PATCH 212/220] fix minor asn bug

---
 bbot/modules/report/asn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py
index a89e04fbf..32590c0dd 100644
--- a/bbot/modules/report/asn.py
+++ b/bbot/modules/report/asn.py
@@ -106,7 +106,7 @@ async def get_asn(self, ip, retries=1):
                     continue
                 return res
         self.warning(f"Error retrieving ASN for {ip}")
-        return []
+        return [], ""
 
     async def get_asn_ripe(self, ip):
         url = f"https://stat.ripe.net/data/network-info/data.json?resource={ip}"

From 9db4b8ddbb544b5287f2c2aaeb7702a7d3825e22 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 19:59:52 -0400
Subject: [PATCH 213/220] fix duplicate sslcert messages

---
 bbot/modules/sslcert.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py
index 12339af7b..13b868feb 100644
--- a/bbot/modules/sslcert.py
+++ b/bbot/modules/sslcert.py
@@ -74,7 +74,7 @@ async def handle_event(self, event):
                     f"Skipping Subject Alternate Names (SANs) on {netloc} because number of hostnames ({len(dns_names):,}) exceeds threshold ({abort_threshold})"
                 )
                 dns_names = dns_names[:1] + [n for n in dns_names[1:] if self.scan.in_scope(n)]
-            for event_type, results in (("DNS_NAME", dns_names), ("EMAIL_ADDRESS", emails)):
+            for event_type, results in (("DNS_NAME", set(dns_names)), ("EMAIL_ADDRESS", emails)):
                 for event_data in results:
                     if event_data is not None and event_data != event:
                         self.debug(f"Discovered new {event_type} via SSL certificate parsing: [{event_data}]")

From 1c19d0f1037dedbd5b14c8ac092e610ce6bddbb9 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 20:04:15 -0400
Subject: [PATCH 214/220] fix iis shortnames bug

---
 bbot/modules/iis_shortnames.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index b25d1ee12..51ea113d6 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -67,11 +67,10 @@ async def directory_confirm(self, target, method, url_hint, affirmative_status_c
         directory_confirm_result = await self.helpers.request(
             method=method, url=url, allow_redirects=False, retries=2, timeout=10
         )
-
-        if directory_confirm_result.status_code == affirmative_status_code:
-            return True
-        else:
-            return False
+        if directory_configm_result is not None:
+            if directory_confirm_result.status_code == affirmative_status_code:
+                return True
+        return False
 
     async def duplicate_check(self, target, method, url_hint, affirmative_status_code):
         duplicates = []

From 907f5caa66bda6dcdc0ec2837a9a266d3df8a548 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 28 May 2024 20:41:09 -0400
Subject: [PATCH 215/220] handle long filenames

---
 bbot/core/helpers/misc.py             | 37 ++++++++++++++++++++++++++-
 bbot/core/helpers/web.py              |  2 ++
 bbot/modules/iis_shortnames.py        |  2 +-
 bbot/test/test_step_1/test_helpers.py | 10 ++++++++
 4 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index b77aaa2b7..568d8924a 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -1454,7 +1454,7 @@ def search_dict_values(d, *regexes):
         ...         ]
         ...     }
         ... }
-        >>> url_regexes = re.compile(r'https?://[^\\s<>"]+|www\.[^\\s<>"]+')
+        >>> url_regexes = re.compile(r'https?://[^\\s<>"]+|www\\.[^\\s<>"]+')
         >>> list(search_dict_values(dict_to_search, url_regexes))
         ["https://www.evilcorp.com"]
     """
@@ -2666,3 +2666,38 @@ async def as_completed(coros):
         for task in done:
             tasks.pop(task)
             yield task
+
+
+def truncate_filename(file_path, max_length=255):
+    """
+    Truncate the filename while preserving the file extension to ensure the total path length does not exceed the maximum length.
+
+    Args:
+        file_path (str): The original file path.
+        max_length (int): The maximum allowed length for the total path. Default is 255.
+
+    Returns:
+        pathlib.Path: A new Path object with the truncated filename.
+
+    Raises:
+        ValueError: If the directory path is too long to accommodate any filename within the limit.
+
+    Example:
+        >>> truncate_filename('/path/to/example_long_filename.txt', 20)
+        PosixPath('/path/to/example.txt')
+    """
+    p = Path(file_path)
+    directory, stem, suffix = p.parent, p.stem, p.suffix
+
+    max_filename_length = max_length - len(str(directory)) - len(suffix) - 1  # 1 for the '/' separator
+
+    if max_filename_length <= 0:
+        raise ValueError("The directory path is too long to accommodate any filename within the limit.")
+
+    if len(stem) > max_filename_length:
+        truncated_stem = stem[:max_filename_length]
+    else:
+        truncated_stem = stem
+
+    new_path = directory / (truncated_stem + suffix)
+    return new_path
diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py
index adcebcfee..f560ad791 100644
--- a/bbot/core/helpers/web.py
+++ b/bbot/core/helpers/web.py
@@ -13,6 +13,7 @@
 from httpx._models import Cookies
 from socksio.exceptions import SOCKSError
 
+from bbot.core.helpers.misc import truncate_filename
 from bbot.core.errors import WordlistError, CurlError
 from bbot.core.helpers.ratelimiter import RateLimiter
 
@@ -258,6 +259,7 @@ async def download(self, url, **kwargs):
         """
         success = False
         filename = kwargs.pop("filename", self.parent_helper.cache_filename(url))
+        filename = truncate_filename(filename)
         follow_redirects = kwargs.pop("follow_redirects", True)
         max_size = kwargs.pop("max_size", None)
         warn = kwargs.pop("warn", True)
diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py
index 51ea113d6..86a98dc86 100644
--- a/bbot/modules/iis_shortnames.py
+++ b/bbot/modules/iis_shortnames.py
@@ -67,7 +67,7 @@ async def directory_confirm(self, target, method, url_hint, affirmative_status_c
         directory_confirm_result = await self.helpers.request(
             method=method, url=url, allow_redirects=False, retries=2, timeout=10
         )
-        if directory_configm_result is not None:
+        if directory_confirm_result is not None:
             if directory_confirm_result.status_code == affirmative_status_code:
                 return True
         return False
diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index c8045e595..8f515961d 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -469,6 +469,16 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https
         await helpers.wordlist("/tmp/a9pseoysadf/asdkgjaosidf")
     test_file.unlink()
 
+    # filename truncation
+    super_long_filename = "/tmp/" + ("a" * 1024) + ".txt"
+    with pytest.raises(OSError):
+        with open(super_long_filename, "w") as f:
+            f.write("wat")
+    truncated_filename = helpers.truncate_filename(super_long_filename)
+    with open(truncated_filename, "w") as f:
+        f.write("wat")
+    truncated_filename.unlink()
+
     # misc DNS helpers
     assert helpers.is_ptr("wsc-11-22-33-44-wat.evilcorp.com") == True
     assert helpers.is_ptr("wsc-11-22-33-wat.evilcorp.com") == False

From 72d2f77416e0ea626fa2709005a97b961efbb242 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Wed, 29 May 2024 00:54:36 -0400
Subject: [PATCH 216/220] rolling baddns version

---
 bbot/modules/baddns.py      | 2 +-
 bbot/modules/baddns_zone.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py
index 9abfebc84..1bfbdec33 100644
--- a/bbot/modules/baddns.py
+++ b/bbot/modules/baddns.py
@@ -20,7 +20,7 @@ class baddns(BaseModule):
         "only_high_confidence": "Do not emit low-confidence or generic detections",
     }
     max_event_handlers = 8
-    deps_pip = ["baddns~=1.1.0"]
+    deps_pip = ["baddns~=1.1.789"]
 
     def select_modules(self):
         selected_modules = []
diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py
index a42fe2e21..2b6ac27ae 100644
--- a/bbot/modules/baddns_zone.py
+++ b/bbot/modules/baddns_zone.py
@@ -18,7 +18,7 @@ class baddns_zone(baddns_module):
         "only_high_confidence": "Do not emit low-confidence or generic detections",
     }
     max_event_handlers = 8
-    deps_pip = ["baddns~=1.1.0"]
+    deps_pip = ["baddns~=1.1.789"]
 
     def select_modules(self):
         selected_modules = []

From 6917afc6a7fda6336aa3dbec2a0816356642984a Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 29 May 2024 11:50:13 -0400
Subject: [PATCH 217/220] revised excavate context

---
 bbot/modules/internal/excavate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index ed9751b1d..25b26717b 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -431,7 +431,7 @@ async def handle_event(self, event):
                             url_event.web_spider_distance = parent_web_spider_distance
                             await self.emit_event(
                                 url_event,
-                                context="{module} extracted {event.type}: {event.data} from redirect destination (Location header)",
+                                context="{module} looked in \"Location\" header and found {event.type}: {event.data}",
                             )
                     else:
                         self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}")

From 325d2edf67de295d9151a58e965d0bf003ffba0e Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Wed, 29 May 2024 12:15:51 -0400
Subject: [PATCH 218/220] blacked

---
 bbot/modules/internal/excavate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 25b26717b..eef4ab26a 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -431,7 +431,7 @@ async def handle_event(self, event):
                             url_event.web_spider_distance = parent_web_spider_distance
                             await self.emit_event(
                                 url_event,
-                                context="{module} looked in \"Location\" header and found {event.type}: {event.data}",
+                                context='{module} looked in "Location" header and found {event.type}: {event.data}',
                             )
                     else:
                         self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}")

From 79c59ed66eb675e1aaf91e1ebc24f60a6b46c830 Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Thu, 30 May 2024 12:17:19 -0400
Subject: [PATCH 219/220] potential

---
 bbot/modules/bypass403.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py
index 38e8b570f..4f3b51789 100644
--- a/bbot/modules/bypass403.py
+++ b/bbot/modules/bypass403.py
@@ -147,7 +147,7 @@ async def handle_event(self, event):
                 },
                 "FINDING",
                 parent=event,
-                context=f"{{module}} discovered multiple 403 bypasses ({{event.type}}) for {event.data}",
+                context=f"{{module}} discovered multiple potential 403 bypasses ({{event.type}}) for {event.data}",
             )
         else:
             for description in results:
@@ -155,7 +155,7 @@ async def handle_event(self, event):
                     {"description": description, "host": str(event.host), "url": event.data},
                     "FINDING",
                     parent=event,
-                    context=f"{{module}} discovered 403 bypass ({{event.type}}) for {event.data}",
+                    context=f"{{module}} discovered potential 403 bypass ({{event.type}}) for {event.data}",
                 )
 
     # When a WAF-check helper is available in the future, we will convert to HTTP_RESPONSE and check for the WAF string here.

From b229d4520f419e022fdda3a63566a7b21c35e4ab Mon Sep 17 00:00:00 2001
From: TheTechromancer <thetechromancer@protonmail.com>
Date: Tue, 18 Jun 2024 10:31:57 -0400
Subject: [PATCH 220/220] don't shuffle portscan's incoming queue

---
 bbot/modules/portscan.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py
index 6a971a569..7a18306e9 100644
--- a/bbot/modules/portscan.py
+++ b/bbot/modules/portscan.py
@@ -42,6 +42,7 @@ class portscan(BaseModule):
     }
     deps_shared = ["masscan"]
     batch_size = 1000000
+    _shuffle_incoming_queue = False
 
     async def setup(self):
         self.top_ports = self.config.get("top_ports", 100)