From 5c5f2f6ef69480bb26fdff1e229bb4253df8cc97 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Wed, 28 Aug 2024 17:10:57 -0400
Subject: [PATCH 01/12] WIP DNS optimizations

---
 bbot/modules/internal/dnsresolve.py | 372 ++++++++++------------------
 1 file changed, 124 insertions(+), 248 deletions(-)

diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index 42ec8cf941..8d1469a306 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -12,10 +12,13 @@
 class DNSResolve(InterceptModule):
     """
     TODO:
-        - scrap event cache in favor of the parent backtracking method
+        - scrap event cache in favor of the parent backtracking method (actual event should have all the information)
         - don't duplicate resolution on the same host
         - clean up wildcard checking to only happen once, and re-emit/abort if one is detected
-        - same thing with main_host_event. we should never be processing two events - only one.
+            - same thing with main_host_event. we should never be processing two events - only one.
+        - do not emit any hosts/children/raw until after scope checks
+            - and only emit them if they're inside scope distance
+            - order: A/AAAA --> scope check --> then rest?
     """
 
     watched_events = ["*"]
@@ -40,6 +43,8 @@ async def setup(self):
             return None, "DNS resolution is disabled in the config"
 
         self.minimal = self.dns_config.get("minimal", False)
+        self.minimal_rdtypes = ("A", "AAAA", "CNAME")
+        self.non_minimal_rdtypes = [t for t in all_rdtypes if t not in self.minimal_rdtypes]
         self.dns_search_distance = max(0, int(self.dns_config.get("search_distance", 1)))
         self._emit_raw_records = None
 
@@ -51,268 +56,122 @@ async def setup(self):
 
         return True
 
-    @property
-    def _dns_search_distance(self):
-        return max(self.scan.scope_search_distance, self.dns_search_distance)
-
-    @property
-    def emit_raw_records(self):
-        if self._emit_raw_records is None:
-            watching_raw_records = any(
-                ["RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()]
-            )
-            omitted_event_types = self.scan.config.get("omit_event_types", [])
-            omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types
-            self._emit_raw_records = watching_raw_records or not omit_raw_records
-        return self._emit_raw_records
-
     async def filter_event(self, event):
         if (not event.host) or (event.type in ("IP_RANGE",)):
             return False, "event does not have host attribute"
         return True
 
     async def handle_event(self, event, **kwargs):
+        raw_records = {}
         event_is_ip = self.helpers.is_ip(event.host)
-        event_host = str(event.host)
-        event_host_hash = hash(event_host)
-
-        async with self._event_cache_locks.lock(event_host_hash):
-            # first thing we do is check for wildcards
-            if not event_is_ip:
-                if event.scope_distance <= self.scan.scope_search_distance:
-                    await self.handle_wildcard_event(event)
-
-        event_host = str(event.host)
-        event_host_hash = hash(event_host)
-
-        # we do DNS resolution inside a lock to make sure we don't duplicate work
-        # once the resolution happens, it will be cached so it doesn't need to happen again
-        async with self._event_cache_locks.lock(event_host_hash):
-            try:
-                # try to get from cache
-                # the "main host event" is the original parent IP_ADDRESS or DNS_NAME
-                main_host_event, dns_tags, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash]
-                # dns_tags, dns_children, event_whitelisted, event_blacklisted = self._event_cache[event_host_hash]
-            except KeyError:
-
-                main_host_event, dns_tags, event_whitelisted, event_blacklisted, raw_record_events = (
-                    await self.resolve_event(event)
-                )
-
-                # if we're not blacklisted and we haven't already done it, emit the main host event and all its raw records
-                main_host_resolved = getattr(main_host_event, "_resolved", False)
-                if not event_blacklisted and not main_host_resolved:
-                    if event_whitelisted:
-                        self.debug(
-                            f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)"
-                        )
-                        main_host_event.scope_distance = 0
-                        await self.handle_wildcard_event(main_host_event)
-
-                    in_dns_scope = -1 < main_host_event.scope_distance < self._dns_search_distance
+        main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event)
 
-                    if event != main_host_event:
-                        await self.emit_event(main_host_event)
-                    for raw_record_event in raw_record_events:
-                        await self.emit_event(raw_record_event)
-
-                    # kill runaway DNS chains
-                    dns_resolve_distance = getattr(event, "dns_resolve_distance", 0)
-                    if dns_resolve_distance >= self.helpers.dns.runaway_limit:
-                        self.debug(
-                            f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})"
-                        )
-                        main_host_event.dns_children = {}
-
-                    # emit DNS children
-                    if not self.minimal:
-                        in_dns_scope = -1 < event.scope_distance < self._dns_search_distance
-                        for rdtype, records in main_host_event.dns_children.items():
-                            module = self._make_dummy_module(rdtype)
-                            for record in records:
-                                parents = main_host_event.get_parents()
-                                for e in parents:
-                                    e_is_host = e.type in ("DNS_NAME", "IP_ADDRESS")
-                                    e_parent_matches = str(e.parent.host) == str(main_host_event.host)
-                                    e_host_matches = str(e.data) == str(record)
-                                    e_module_matches = str(e.module) == str(module)
-                                    if e_is_host and e_parent_matches and e_host_matches and e_module_matches:
-                                        self.trace(
-                                            f"TRYING TO EMIT ALREADY-EMITTED {record}:{rdtype} CHILD OF {main_host_event}, parents: {parents}"
-                                        )
-                                        return
-                                try:
-                                    child_event = self.scan.make_event(
-                                        record, "DNS_NAME", module=module, parent=main_host_event
-                                    )
-                                    child_event.discovery_context = f"{rdtype} record for {event.host} contains {child_event.type}: {child_event.host}"
-                                    # if it's a hostname and it's only one hop away, mark it as affiliate
-                                    if child_event.type == "DNS_NAME" and child_event.scope_distance == 1:
-                                        child_event.add_tag("affiliate")
-                                    if in_dns_scope or self.preset.in_scope(child_event):
-                                        self.debug(f"Queueing DNS child for {event}: {child_event}")
-                                        await self.emit_event(child_event)
-                                except ValidationError as e:
-                                    self.warning(
-                                        f'Event validation failed for DNS child of {main_host_event}: "{record}" ({rdtype}): {e}'
-                                    )
-
-                # mark the host as resolved
-                main_host_event._resolved = True
-
-                # store results in cache
-                self._event_cache[event_host_hash] = main_host_event, dns_tags, event_whitelisted, event_blacklisted
+        # minimal resolution - first, we resolve A/AAAA records for scope purposes
+        if new_event or event is main_host_event:
+            if event_is_ip:
+                basic_records = await self.resolve_event(main_host_event, types=("PTR",))
+            else:
+                basic_records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes)
+                # are any of its IPs whitelisted/blacklisted?
+                whitelisted, blacklisted = self.check_scope(main_host_event)
+                if whitelisted and event.scope_distance > 0:
+                    self.debug(
+                        f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)"
+                    )
+                    main_host_event.scope_distance = 0
+            raw_records.update(basic_records)
 
         # abort if the event resolves to something blacklisted
-        if event_blacklisted:
-            return False, f"it has a blacklisted DNS record"
-
-        # if the event resolves to an in-scope IP, set its scope distance to 0
-        if event_whitelisted:
-            self.debug(f"Making {event} in-scope because it resolves to an in-scope resource")
-            event.scope_distance = 0
-            await self.handle_wildcard_event(event)
-
-        # transfer resolved hosts
-        event._resolved_hosts = main_host_event._resolved_hosts
-
-        # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
-        if event.type == "DNS_NAME" and "unresolved" in event.tags:
-            event.type = "DNS_NAME_UNRESOLVED"
-
-    async def resolve_event(self, event):
-        dns_tags = set()
-        event_whitelisted = False
-        event_blacklisted = False
-
-        main_host_event = self.get_dns_parent(event)
-        event_host = str(event.host)
-        event_is_ip = self.helpers.is_ip(event.host)
-
-        rdtypes_to_resolve = ()
-        if event_is_ip:
-            if not self.minimal:
-                rdtypes_to_resolve = ("PTR",)
+        if blacklisted:
+            return False, "it has a blacklisted DNS record"
+
+        # are we within our dns search distance?
+        within_dns_distance = main_host_event.scope_distance <= self._dns_search_distance
+        within_scope_distance = main_host_event.scope_distance <= self.scan.scope_search_distance
+
+        # if so, resolve the rest of our records
+        if not event_is_ip:
+            if (not self.minimal) and within_dns_distance:
+                records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes)
+                raw_records.update(records)
+            # check for wildcards if we're within the scan's search distance
+            if new_event and within_scope_distance:
+                await self.handle_wildcard_event(main_host_event)
+
+        # kill runaway DNS chains
+        # TODO: test this
+        dns_resolve_distance = getattr(event, "dns_resolve_distance", 0)
+        runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit
+        if runaway_dns:
+            self.debug(
+                f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})"
+            )
         else:
-            if self.minimal:
-                rdtypes_to_resolve = ("A", "AAAA", "CNAME")
-            else:
-                rdtypes_to_resolve = all_rdtypes
-
-        # if missing from cache, do DNS resolution
-        queries = [(event_host, rdtype) for rdtype in rdtypes_to_resolve]
-        error_rdtypes = []
-        raw_record_events = []
-        async for (query, rdtype), (answer, errors) in self.helpers.dns.resolve_raw_batch(queries):
-            if self.emit_raw_records and rdtype not in ("A", "AAAA", "CNAME", "PTR"):
-                raw_record_event = self.make_event(
-                    {"host": str(event_host), "type": rdtype, "answer": answer.to_text()},
-                    "RAW_DNS_RECORD",
-                    parent=main_host_event,
-                    tags=[f"{rdtype.lower()}-record"],
-                    context=f"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}",
-                )
-                raw_record_events.append(raw_record_event)
-            if errors:
-                error_rdtypes.append(rdtype)
-            dns_tags.add(f"{rdtype.lower()}-record")
-            for _rdtype, host in extract_targets(answer):
-                try:
-                    main_host_event.dns_children[_rdtype].add(host)
-                except KeyError:
-                    main_host_event.dns_children[_rdtype] = {host}
-
-        # if there were dns resolution errors, notify the user with tags
-        for rdtype in error_rdtypes:
-            if rdtype not in main_host_event.dns_children:
-                dns_tags.add(f"{rdtype.lower()}-error")
-
-        # if there weren't any DNS children and it's not an IP address, tag as unresolved
-        if not main_host_event.dns_children and not event_is_ip:
-            dns_tags.add("unresolved")
-
-        # check DNS children against whitelists and blacklists
-        for rdtype, children in main_host_event.dns_children.items():
-            if event_blacklisted:
-                break
-            for host in children:
-                # whitelisting / blacklisting based on resolved hosts
-                if rdtype in ("A", "AAAA", "CNAME"):
-                    # having a CNAME to an in-scope resource doesn't make you in-scope
-                    if (not event_whitelisted) and rdtype != "CNAME":
+            if within_dns_distance:
+                pass
+                # emit dns children
+                # emit raw records
+                # emit host event
+
+        # update host event --> event
+
+    def check_scope(self, event):
+        whitelisted = False
+        blacklisted = False
+        dns_children = getattr(event, "dns_children", {})
+        for rdtype in ("A", "AAAA", "CNAME"):
+            hosts = dns_children.get(rdtype, [])
+            # update resolved hosts
+            event.resolved_hosts.update(hosts)
+            for host in hosts:
+                # having a CNAME to an in-scope resource doesn't make you in-scope
+                if rdtype != "CNAME":
+                    if not whitelisted:
                         with suppress(ValidationError):
                             if self.scan.whitelisted(host):
-                                event_whitelisted = True
-                                dns_tags.add(f"dns-whitelisted-{rdtype.lower()}")
-                    # CNAME to a blacklisted resource, means you're blacklisted
+                                whitelisted = True
+                                event.add_tag(f"dns-whitelisted-{rdtype}")
+                if not blacklisted:
                     with suppress(ValidationError):
                         if self.scan.blacklisted(host):
-                            dns_tags.add("blacklisted")
-                            dns_tags.add(f"dns-blacklisted-{rdtype.lower()}")
-                            event_blacklisted = True
-                            event_whitelisted = False
-                            break
-
-                # check for private IPs
+                            blacklisted = True
+                            event.add_tag("blacklisted")
+                            event.add_tag(f"dns-blacklisted-{rdtype}")
+        if blacklisted:
+            whitelisted = False
+        return whitelisted, blacklisted
+
+    async def resolve_event(self, event, types):
+        raw_records = {}
+        event_host = str(event.host)
+        queries = [(event_host, rdtype) for rdtype in types]
+        dns_errors = {}
+        async for (query, rdtype), (answer, errors) in self.helpers.dns.resolve_raw_batch(queries):
+            try:
+                dns_errors[rdtype].update(errors)
+            except KeyError:
+                dns_errors[rdtype] = set(errors)
+            # raw dnspython objects
+            try:
+                raw_records[rdtype].add(answer)
+            except KeyError:
+                raw_records[rdtype] = {answer}
+            # hosts
+            for _rdtype, host in extract_targets(answer):
+                event.add_tag(f"{_rdtype}-record")
                 try:
-                    ip = ipaddress.ip_address(host)
-                    if ip.is_private:
-                        dns_tags.add("private-ip")
-                except ValueError:
-                    continue
-
-        # add DNS tags to main host
-        for tag in dns_tags:
-            main_host_event.add_tag(tag)
-
-        # set resolved_hosts attribute
-        for rdtype, children in main_host_event.dns_children.items():
-            if rdtype in ("A", "AAAA", "CNAME"):
-                for host in children:
-                    main_host_event._resolved_hosts.add(host)
-
-        return main_host_event, dns_tags, event_whitelisted, event_blacklisted, raw_record_events
+                    event.dns_children[_rdtype].add(host)
+                except KeyError:
+                    event.dns_children[_rdtype] = {host}
+        # tag event with errors
+        for rdtype, errors in dns_errors.items():
+            # only consider it an error if there weren't any results for that rdtype
+            if errors and not rdtype in event.dns_children:
+                event.add_tag(f"{rdtype}-error")
+        return raw_records
 
     async def handle_wildcard_event(self, event):
-        self.debug(f"Entering handle_wildcard_event({event})")
-        try:
-            event_host = str(event.host)
-            # check if the dns name itself is a wildcard entry
-            wildcard_rdtypes = await self.helpers.is_wildcard(event_host)
-            for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items():
-                if is_wildcard == False:
-                    continue
-                elif is_wildcard == True:
-                    event.add_tag("wildcard")
-                    wildcard_tag = "wildcard"
-                elif is_wildcard == None:
-                    wildcard_tag = "error"
-
-                event.add_tag(f"{rdtype.lower()}-{wildcard_tag}")
-
-            # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
-            if wildcard_rdtypes and not "target" in event.tags:
-                # these are the rdtypes that have wildcards
-                wildcard_rdtypes_set = set(wildcard_rdtypes)
-                # consider the event a full wildcard if all its records are wildcards
-                event_is_wildcard = False
-                if wildcard_rdtypes_set:
-                    event_is_wildcard = all(r[0] == True for r in wildcard_rdtypes.values())
-
-                if event_is_wildcard:
-                    if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."):
-                        wildcard_parent = self.helpers.parent_domain(event_host)
-                        for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items():
-                            if _is_wildcard:
-                                wildcard_parent = _parent_domain
-                                break
-                        wildcard_data = f"_wildcard.{wildcard_parent}"
-                        if wildcard_data != event.data:
-                            self.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"')
-                            event.data = wildcard_data
-
-        finally:
-            self.debug(f"Finished handle_wildcard_event({event})")
+        pass
 
     def get_dns_parent(self, event):
         """
@@ -320,7 +179,9 @@ def get_dns_parent(self, event):
         """
         for parent in event.get_parents(include_self=True):
             if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"):
-                return parent
+                blacklisted = any(t.startswith("dns-blacklisted-") for t in parent.tags)
+                whitelisted = any(t.startswith("dns-whitelisted-") for t in parent.tags)
+                return parent, whitelisted, blacklisted, False
         tags = set()
         if "target" in event.tags:
             tags.add("target")
@@ -331,7 +192,22 @@ def get_dns_parent(self, event):
             parent=event,
             context="{event.parent.type} has host {event.type}: {event.host}",
             tags=tags,
-        )
+        ), None, None, True
+
+    @property
+    def emit_raw_records(self):
+        if self._emit_raw_records is None:
+            watching_raw_records = any(
+                ["RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()]
+            )
+            omitted_event_types = self.scan.config.get("omit_event_types", [])
+            omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types
+            self._emit_raw_records = watching_raw_records or not omit_raw_records
+        return self._emit_raw_records
+
+    @property
+    def _dns_search_distance(self):
+        return max(self.scan.scope_search_distance, self.dns_search_distance)
 
     def _make_dummy_module(self, name):
         try:

From d3ad8fe1d4d3a445cc09e6ac525818d9b6bddb18 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Thu, 29 Aug 2024 17:06:21 -0400
Subject: [PATCH 02/12] more DNS WIP

---
 bbot/core/event/base.py             |   1 +
 bbot/core/helpers/dns/dns.py        |  20 ++-
 bbot/core/helpers/dns/engine.py     | 252 ++++++++++++++-------------
 bbot/defaults.yml                   |   2 +-
 bbot/errors.py                      |   8 -
 bbot/modules/internal/dnsresolve.py | 257 ++++++++++++++++++----------
 bbot/test/bbot_fixtures.py          |  12 +-
 bbot/test/test_step_1/test_dns.py   |  23 +--
 8 files changed, 332 insertions(+), 243 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 3eb10625f7..94df01cf57 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -171,6 +171,7 @@ def __init__(
         self._module_priority = None
         self._resolved_hosts = set()
         self.dns_children = dict()
+        self.raw_dns_records = dict()
         self._discovery_context = ""
         self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
         self.web_spider_distance = 0
diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py
index 07f5621323..7574740153 100644
--- a/bbot/core/helpers/dns/dns.py
+++ b/bbot/core/helpers/dns/dns.py
@@ -117,8 +117,11 @@ def brute(self):
             self._brute = DNSBrute(self.parent_helper)
         return self._brute
 
-    @async_cachedmethod(lambda self: self._is_wildcard_cache)
-    async def is_wildcard(self, query, ips=None, rdtype=None):
+    @async_cachedmethod(
+        lambda self: self._is_wildcard_cache,
+        key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes))),
+    )
+    async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         """
         Use this method to check whether a *host* is a wildcard entry
 
@@ -150,9 +153,6 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
         Note:
             `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)
         """
-        if [ips, rdtype].count(None) == 1:
-            raise ValueError("Both ips and rdtype must be specified")
-
         query = self._wildcard_prevalidation(query)
         if not query:
             return {}
@@ -161,15 +161,17 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
         if is_domain(query):
             return {}
 
-        return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype)
+        return await self.run_and_return("is_wildcard", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records)
 
-    @async_cachedmethod(lambda self: self._is_wildcard_domain_cache)
-    async def is_wildcard_domain(self, domain, log_info=False):
+    @async_cachedmethod(
+        lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes)))
+    )
+    async def is_wildcard_domain(self, domain, rdtypes):
         domain = self._wildcard_prevalidation(domain)
         if not domain:
             return {}
 
-        return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False)
+        return await self.run_and_return("is_wildcard_domain", domain=domain, rdtypes=rdtypes)
 
     def _wildcard_prevalidation(self, host):
         if self.wildcard_disable:
diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py
index 8a41c7c8ea..e72b05b80b 100644
--- a/bbot/core/helpers/dns/engine.py
+++ b/bbot/core/helpers/dns/engine.py
@@ -7,7 +7,6 @@
 from cachetools import LRUCache
 from contextlib import suppress
 
-from bbot.errors import DNSWildcardBreak
 from bbot.core.engine import EngineServer
 from bbot.core.helpers.async_helpers import NamedLock
 from bbot.core.helpers.dns.helpers import extract_targets
@@ -16,7 +15,6 @@
     rand_string,
     parent_domain,
     domain_parents,
-    clean_dns_record,
 )
 
 
@@ -361,8 +359,7 @@ async def resolve_raw_batch(self, queries, threads=10, **kwargs):
         ):
             query = args[0]
             rdtype = kwargs["type"]
-            for answer in answers:
-                yield ((query, rdtype), (answer, errors))
+            yield ((query, rdtype), (answers, errors))
 
     async def _catch(self, callback, *args, **kwargs):
         """
@@ -397,7 +394,7 @@ async def _catch(self, callback, *args, **kwargs):
             self.log.trace(traceback.format_exc())
         return []
 
-    async def is_wildcard(self, query, ips=None, rdtype=None):
+    async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         """
         Use this method to check whether a *host* is a wildcard entry
 
@@ -407,8 +404,8 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
 
         Args:
             query (str): The hostname to check for a wildcard entry.
-            ips (list, optional): List of IPs to compare against, typically obtained from a previous DNS resolution of the query.
-            rdtype (str, optional): The DNS record type (e.g., "A", "AAAA") to consider during the check.
+            rdtypes (list): The DNS record type (e.g., "A", "AAAA") to consider during the check.
+            raw_dns_records (dict, optional): Dictionary of {rdtype: [answer1, answer2, ...], ...} containing raw dnspython answers for the query.
 
         Returns:
             dict: A dictionary indicating if the query is a wildcard for each checked DNS record type.
@@ -420,104 +417,104 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
             ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified.
 
         Examples:
-            >>> is_wildcard("www.github.io")
-            {"A": (True, "github.io"), "AAAA": (True, "github.io")}
+            >>> is_wildcard("www.github.io", rdtypes=["A", "AAAA", "MX"])
+            {"A": (True, "github.io"), "AAAA": (True, "github.io"), "MX": (False, "github.io")}
 
-            >>> is_wildcard("www.evilcorp.com", ips=["93.184.216.34"], rdtype="A")
+            >>> is_wildcard("www.evilcorp.com", rdtypes=["A"])
             {"A": (False, "evilcorp.com")}
 
         Note:
             `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)
         """
-        result = {}
+        if isinstance(rdtypes, str):
+            rdtypes = [rdtypes]
 
-        parent = parent_domain(query)
-        parents = list(domain_parents(query))
+        result = {}
 
-        if rdtype is not None:
-            if isinstance(rdtype, str):
-                rdtype = [rdtype]
-            rdtypes_to_check = rdtype
-        else:
-            rdtypes_to_check = all_rdtypes
-
-        query_baseline = dict()
-        # if the caller hasn't already done the work of resolving the IPs
-        if ips is None:
-            # then resolve the query for all rdtypes
-            queries = [(query, t) for t in rdtypes_to_check]
-            async for (query, _rdtype), (answers, errors) in self.resolve_raw_batch(queries):
-                answers = extract_targets(answers)
+        # if the work of resolving hasn't been done yet, do it
+        if raw_dns_records is None:
+            raw_dns_records = {}
+            queries = [(query, rdtype) for rdtype in rdtypes]
+            async for (_, rdtype), (answers, errors) in self.resolve_raw_batch(queries):
                 if answers:
-                    query_baseline[_rdtype] = set([a[1] for a in answers])
+                    for answer in answers:
+                        try:
+                            raw_dns_records[rdtype].add(answer)
+                        except KeyError:
+                            raw_dns_records[rdtype] = {answer}
                 else:
                     if errors:
-                        self.debug(f"Failed to resolve {query} ({_rdtype}) during wildcard detection")
-                        result[_rdtype] = (None, parent)
-                        continue
-        else:
-            # otherwise, we can skip all that
-            cleaned_ips = set([clean_dns_record(ip) for ip in ips])
-            if not cleaned_ips:
-                raise ValueError("Valid IPs must be specified")
-            query_baseline[rdtype] = cleaned_ips
-        if not query_baseline:
+                        self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection")
+                        result[rdtype] = (None, query)
+
+        # clean + process the raw records into a baseline
+        baseline = {}
+        baseline_raw = {}
+        for rdtype, answers in raw_dns_records.items():
+            for answer in answers:
+                text_answer = answer.to_text()
+                try:
+                    baseline_raw[rdtype].add(text_answer)
+                except KeyError:
+                    baseline_raw[rdtype] = {text_answer}
+                for _, host in extract_targets(answer):
+                    try:
+                        baseline[rdtype].add(host)
+                    except KeyError:
+                        baseline[rdtype] = {host}
+
+        # if it's unresolved, it's a big nope
+        if not raw_dns_records:
             return result
 
         # once we've resolved the base query and have IP addresses to work with
         # we can compare the IPs to the ones we have on file for wildcards
 
         # for every parent domain, starting with the shortest
-        try:
-            for host in parents[::-1]:
-                # make sure we've checked that domain for wildcards
-                await self.is_wildcard_domain(host)
-
-                # for every rdtype
-                for _rdtype in list(query_baseline):
-                    # get the IPs from above
-                    query_ips = query_baseline.get(_rdtype, set())
-                    host_hash = hash(host)
-
-                    if host_hash in self._wildcard_cache:
-                        # then get its IPs from our wildcard cache
-                        wildcard_rdtypes = self._wildcard_cache[host_hash]
-
-                        # then check to see if our IPs match the wildcard ones
-                        if _rdtype in wildcard_rdtypes:
-                            wildcard_ips = wildcard_rdtypes[_rdtype]
-                            # if our IPs match the wildcard ones, then ladies and gentlemen we have a wildcard
-                            is_wildcard = any(r in wildcard_ips for r in query_ips)
-
-                            if is_wildcard and not result.get(_rdtype, (None, None))[0] is True:
-                                result[_rdtype] = (True, host)
-
-                    # if we've reached a point where the dns name is a complete wildcard, class can be dismissed early
-                    base_query_rdtypes = set(query_baseline)
-                    wildcard_rdtypes_set = set([k for k, v in result.items() if v[0] is True])
-                    if base_query_rdtypes and wildcard_rdtypes_set and base_query_rdtypes == wildcard_rdtypes_set:
-                        self.log.debug(
-                            f"Breaking from wildcard detection for {query} at {host} because base query rdtypes ({base_query_rdtypes}) == wildcard rdtypes ({wildcard_rdtypes_set})"
-                        )
-                        raise DNSWildcardBreak()
-
-        except DNSWildcardBreak:
-            pass
-
-        for _rdtype, answers in query_baseline.items():
-            if answers and _rdtype not in result:
-                result[_rdtype] = (False, query)
+        parents = list(domain_parents(query))
+        for parent in parents[::-1]:
+
+            # check if the parent domain is set up with wildcards
+            wildcard_results = await self.is_wildcard_domain(parent, rdtypes)
+
+            # for every rdtype
+            for rdtype in list(baseline_raw):
+                # skip if we already found a wildcard for this rdtype
+                if rdtype in result:
+                    continue
+
+                # get our baseline IPs from above
+                _baseline = baseline.get(rdtype, set())
+                _baseline_raw = baseline_raw.get(rdtype, set())
+
+                wildcards = wildcard_results.get(rdtype, None)
+                if wildcards is None:
+                    continue
+                wildcards, wildcard_raw = wildcards
+
+                # check if any of our baseline IPs are in the wildcard results
+                is_wildcard = any(r in wildcards for r in _baseline)
+                is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw)
+
+                # if there are any matches, we have a wildcard
+                if is_wildcard or is_wildcard_raw:
+                    result[rdtype] = (True, query)
+
+        # any rdtype that wasn't a wildcard, mark it as False
+        for rdtype, answers in baseline_raw.items():
+            if answers and rdtype not in result:
+                result[rdtype] = (False, query)
 
         return result
 
-    async def is_wildcard_domain(self, domain, log_info=False):
+    async def is_wildcard_domain(self, domain, rdtypes):
         """
         Check whether a given host or its children make use of wildcard DNS entries. Wildcard DNS can have
         various implications, particularly in subdomain enumeration and subdomain takeovers.
 
         Args:
             domain (str): The domain to check for wildcard DNS entries.
-            log_info (bool, optional): Whether to log the result of the check. Defaults to False.
+            rdtypes (list): Which DNS record types to check.
 
         Returns:
             dict: A dictionary where the keys are the parent domains that have wildcard DNS entries,
@@ -531,60 +528,75 @@ async def is_wildcard_domain(self, domain, log_info=False):
             >>> is_wildcard_domain("example.com")
             {}
         """
-        wildcard_domain_results = {}
-
-        rdtypes_to_check = set(all_rdtypes)
-
+        if isinstance(rdtypes, str):
+            rdtypes = [rdtypes]
+        rdtypes = set(rdtypes)
+        wildcard_results = {}
         # make a list of its parents
         parents = list(domain_parents(domain, include_self=True))
         # and check each of them, beginning with the highest parent (i.e. the root domain)
         for i, host in enumerate(parents[::-1]):
-            # have we checked this host before?
-            host_hash = hash(host)
-            async with self._wildcard_lock.lock(host_hash):
-                # if we've seen this host before
-                if host_hash in self._wildcard_cache:
-                    wildcard_domain_results[host] = self._wildcard_cache[host_hash]
-                    continue
+            queries = [((host, rdtype), {}) for rdtype in rdtypes]
+            async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool(
+                self._is_wildcard_zone, args_kwargs=queries
+            ):
+                # if we hit a wildcard, we can skip this rdtype from now on
+                if results_raw:
+                    rdtypes.remove(rdtype)
+                    wildcard_results[rdtype] = results, results_raw
+
+        if wildcard_results:
+            wildcard_rdtypes_str = ",".join(sorted(wildcard_results))
+            self.log.info(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}")
+        else:
+            self.log.verbose(f"Finished checking {host}, it is not a wildcard")
 
-                self.log.verbose(f"Checking if {host} is a wildcard")
+        return wildcard_results
 
-                # determine if this is a wildcard domain
+    async def _is_wildcard_zone(self, host, rdtype):
+        """
+        Check whether a specific DNS zone+rdtype has a wildcard configuration
+        """
+        rdtype = rdtype.upper()
 
-                # resolve a bunch of random subdomains of the same parent
-                is_wildcard = False
-                wildcard_results = dict()
+        # have we checked this host before?
+        host_hash = hash(f"{host}:{rdtype}")
+        async with self._wildcard_lock.lock(host_hash):
+            # if we've seen this host before
+            try:
+                wildcard_results, wildcard_results_raw = self._wildcard_cache[host_hash]
+                self.debug(f"Got {host}:{rdtype} from cache")
+            except KeyError:
+                wildcard_results = set()
+                wildcard_results_raw = set()
+                self.debug(f"Checking if {host}:{rdtype} is a wildcard")
 
+                # determine if this is a wildcard domain
+                # resolve a bunch of random subdomains of the same parent
                 rand_queries = []
-                for rdtype in rdtypes_to_check:
-                    for _ in range(self.wildcard_tests):
-                        rand_query = f"{rand_string(digits=False, length=10)}.{host}"
-                        rand_queries.append((rand_query, rdtype))
+                for _ in range(self.wildcard_tests):
+                    rand_query = f"{rand_string(digits=False, length=10)}.{host}"
+                    rand_queries.append((rand_query, rdtype))
 
                 async for (query, rdtype), (answers, errors) in self.resolve_raw_batch(rand_queries, use_cache=False):
-                    answers = extract_targets(answers)
-                    if answers:
-                        is_wildcard = True
-                        if not rdtype in wildcard_results:
-                            wildcard_results[rdtype] = set()
-                        wildcard_results[rdtype].update(set(a[1] for a in answers))
-                        # we know this rdtype is a wildcard
-                        # so we don't need to check it anymore
-                        with suppress(KeyError):
-                            rdtypes_to_check.remove(rdtype)
-
-                self._wildcard_cache.update({host_hash: wildcard_results})
-                wildcard_domain_results.update({host: wildcard_results})
-                if is_wildcard:
-                    wildcard_rdtypes_str = ",".join(sorted([t.upper() for t, r in wildcard_results.items() if r]))
-                    log_fn = self.log.verbose
-                    if log_info:
-                        log_fn = self.log.info
-                    log_fn(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}")
+                    for answer in answers:
+                        # consider both the raw record
+                        wildcard_results_raw.add(answer.to_text())
+                        # and all the extracted hosts
+                        for _, t in extract_targets(answer):
+                            wildcard_results.add(t)
+
+                if wildcard_results:
+                    self.debug(f"Finished checking {host}:{rdtype}, it is a wildcard")
                 else:
-                    self.log.verbose(f"Finished checking {host}, it is not a wildcard")
+                    self.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard")
+                self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw
+
+        return wildcard_results, wildcard_results_raw
 
-        return wildcard_domain_results
+    async def _is_wildcard(self, query, rdtypes, dns_children):
+        if isinstance(rdtypes, str):
+            rdtypes = [rdtypes]
 
     @property
     def dns_connectivity_lock(self):
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index 2ce8d42086..f577966723 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -30,7 +30,7 @@ dns:
   # Speed up scan by not creating any new DNS events, and only resolving A and AAAA records
   minimal: false
   # How many instances of the dns module to run concurrently
-  threads: 20
+  threads: 25
   # How many concurrent DNS resolvers to use when brute-forcing
   # (under the hood this is passed through directly to massdns -s)
   brute_threads: 1000
diff --git a/bbot/errors.py b/bbot/errors.py
index 53bdee48c9..a5ab9619b3 100644
--- a/bbot/errors.py
+++ b/bbot/errors.py
@@ -38,14 +38,6 @@ class WordlistError(BBOTError):
     pass
 
 
-class DNSError(BBOTError):
-    pass
-
-
-class DNSWildcardBreak(DNSError):
-    pass
-
-
 class CurlError(BBOTError):
     pass
 
diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index 8d1469a306..8ffbeb60ba 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -1,26 +1,13 @@
 import ipaddress
 from contextlib import suppress
-from cachetools import LFUCache
 
 from bbot.errors import ValidationError
 from bbot.core.helpers.dns.engine import all_rdtypes
-from bbot.core.helpers.async_helpers import NamedLock
 from bbot.modules.base import InterceptModule, BaseModule
 from bbot.core.helpers.dns.helpers import extract_targets
 
 
 class DNSResolve(InterceptModule):
-    """
-    TODO:
-        - scrap event cache in favor of the parent backtracking method (actual event should have all the information)
-        - don't duplicate resolution on the same host
-        - clean up wildcard checking to only happen once, and re-emit/abort if one is detected
-            - same thing with main_host_event. we should never be processing two events - only one.
-        - do not emit any hosts/children/raw until after scope checks
-            - and only emit them if they're inside scope distance
-            - order: A/AAAA --> scope check --> then rest?
-    """
-
     watched_events = ["*"]
     _priority = 1
     scope_distance_modifier = None
@@ -44,15 +31,16 @@ async def setup(self):
 
         self.minimal = self.dns_config.get("minimal", False)
         self.minimal_rdtypes = ("A", "AAAA", "CNAME")
-        self.non_minimal_rdtypes = [t for t in all_rdtypes if t not in self.minimal_rdtypes]
+        if self.minimal:
+            self.non_minimal_rdtypes = ()
+        else:
+            self.non_minimal_rdtypes = tuple([t for t in all_rdtypes if t not in self.minimal_rdtypes])
         self.dns_search_distance = max(0, int(self.dns_config.get("search_distance", 1)))
         self._emit_raw_records = None
 
-        # event resolution cache
-        self._event_cache = LFUCache(maxsize=10000)
-        self._event_cache_locks = NamedLock()
-
         self.host_module = self.HostModule(self.scan)
+        self.children_emitted = set()
+        self.children_emitted_raw = set()
 
         return True
 
@@ -62,58 +50,146 @@ async def filter_event(self, event):
         return True
 
     async def handle_event(self, event, **kwargs):
-        raw_records = {}
         event_is_ip = self.helpers.is_ip(event.host)
+        if event_is_ip:
+            minimal_rdtypes = ("PTR",)
+            non_minimal_rdtypes = ()
+        else:
+            minimal_rdtypes = self.minimal_rdtypes
+            non_minimal_rdtypes = self.non_minimal_rdtypes
+
+        # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event
         main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event)
 
         # minimal resolution - first, we resolve A/AAAA records for scope purposes
         if new_event or event is main_host_event:
-            if event_is_ip:
-                basic_records = await self.resolve_event(main_host_event, types=("PTR",))
-            else:
-                basic_records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes)
-                # are any of its IPs whitelisted/blacklisted?
-                whitelisted, blacklisted = self.check_scope(main_host_event)
-                if whitelisted and event.scope_distance > 0:
-                    self.debug(
-                        f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)"
-                    )
-                    main_host_event.scope_distance = 0
-            raw_records.update(basic_records)
+            await self.resolve_event(main_host_event, types=minimal_rdtypes)
+            # are any of its IPs whitelisted/blacklisted?
+            whitelisted, blacklisted = self.check_scope(main_host_event)
+            if whitelisted and event.scope_distance > 0:
+                self.debug(f"Making {main_host_event} in-scope because it resolves to an in-scope resource (A/AAAA)")
+                main_host_event.scope_distance = 0
 
         # abort if the event resolves to something blacklisted
         if blacklisted:
             return False, "it has a blacklisted DNS record"
 
-        # are we within our dns search distance?
-        within_dns_distance = main_host_event.scope_distance <= self._dns_search_distance
-        within_scope_distance = main_host_event.scope_distance <= self.scan.scope_search_distance
-
-        # if so, resolve the rest of our records
         if not event_is_ip:
-            if (not self.minimal) and within_dns_distance:
-                records = await self.resolve_event(main_host_event, types=self.minimal_rdtypes)
-                raw_records.update(records)
-            # check for wildcards if we're within the scan's search distance
-            if new_event and within_scope_distance:
-                await self.handle_wildcard_event(main_host_event)
-
-        # kill runaway DNS chains
-        # TODO: test this
-        dns_resolve_distance = getattr(event, "dns_resolve_distance", 0)
+            # if we're within our dns search distance, resolve the rest of our records
+            if main_host_event.scope_distance < self._dns_search_distance:
+                await self.resolve_event(main_host_event, types=non_minimal_rdtypes)
+                # check for wildcards if we're within the scan's search distance
+                if (
+                    new_event
+                    and main_host_event.scope_distance <= self.scan.scope_search_distance
+                    and not "domain" in main_host_event.tags
+                ):
+                    await self.handle_wildcard_event(main_host_event)
+
+        dns_resolve_distance = getattr(main_host_event, "dns_resolve_distance", 0)
         runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit
         if runaway_dns:
+            # kill runaway DNS chains
+            # TODO: test this
             self.debug(
                 f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})"
             )
         else:
-            if within_dns_distance:
-                pass
-                # emit dns children
-                # emit raw records
-                # emit host event
+            # emit dns children
+            await self.emit_dns_children(main_host_event)
+            await self.emit_dns_children_raw(main_host_event)
+
+        # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
+        if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags:
+            main_host_event.type = "DNS_NAME_UNRESOLVED"
+
+        # emit the main DNS_NAME or IP_ADDRESS
+        if new_event and event is not main_host_event and main_host_event.scope_distance <= self._dns_search_distance:
+            await self.emit_event(main_host_event)
+
+        # transfer scope distance to event
+        event.scope_distance = main_host_event.scope_distance
+        event._resolved_hosts = main_host_event.resolved_hosts
+
+    async def handle_wildcard_event(self, event):
+        rdtypes = tuple(event.raw_dns_records)
+        wildcard_rdtypes = await self.helpers.is_wildcard(
+            event.host, rdtypes=rdtypes, raw_dns_records=event.raw_dns_records
+        )
+        for rdtype, (is_wildcard, wildcard_host) in wildcard_rdtypes.items():
+            if is_wildcard == False:
+                continue
+            elif is_wildcard == True:
+                event.add_tag("wildcard")
+                wildcard_tag = "wildcard"
+            elif is_wildcard == None:
+                wildcard_tag = "error"
+            event.add_tag(f"{rdtype}-{wildcard_tag}")
 
-        # update host event --> event
+        # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
+        if wildcard_rdtypes and not "target" in event.tags:
+            # these are the rdtypes that have wildcards
+            wildcard_rdtypes_set = set(wildcard_rdtypes)
+            # consider the event a full wildcard if all its records are wildcards
+            event_is_wildcard = False
+            if wildcard_rdtypes_set:
+                event_is_wildcard = all(r[0] == True for r in wildcard_rdtypes.values())
+
+            if event_is_wildcard:
+                if event.type in ("DNS_NAME",) and not "_wildcard" in event.data.split("."):
+                    wildcard_parent = self.helpers.parent_domain(event.host)
+                    for rdtype, (_is_wildcard, _parent_domain) in wildcard_rdtypes.items():
+                        if _is_wildcard:
+                            wildcard_parent = _parent_domain
+                            break
+                    wildcard_data = f"_wildcard.{wildcard_parent}"
+                    if wildcard_data != event.data:
+                        self.debug(f'Wildcard detected, changing event.data "{event.data}" --> "{wildcard_data}"')
+                        event.data = wildcard_data
+
+    async def emit_dns_children(self, event):
+        for rdtype, children in event.dns_children.items():
+            module = self._make_dummy_module(rdtype)
+            for child_host in children:
+                try:
+                    child_event = self.scan.make_event(
+                        child_host,
+                        "DNS_NAME",
+                        module=module,
+                        parent=event,
+                        context=f"{rdtype} record for {event.host} contains {{event.type}}: {{event.host}}",
+                    )
+                except ValidationError as e:
+                    self.warning(f'Event validation failed for DNS child of {event}: "{child_host}" ({rdtype}): {e}')
+                    continue
+
+                child_hash = hash(f"{event.host}:{module}:{child_host}")
+                # if we haven't emitted this one before
+                if child_hash not in self.children_emitted:
+                    # and it's either in-scope or inside our dns search distance
+                    if self.preset.in_scope(child_host) or child_event.scope_distance <= self._dns_search_distance:
+                        self.children_emitted.add(child_hash)
+                        # if it's a hostname and it's only one hop away, mark it as affiliate
+                        if child_event.type == "DNS_NAME" and child_event.scope_distance == 1:
+                            child_event.add_tag("affiliate")
+                        self.debug(f"Queueing DNS child for {event}: {child_event}")
+                        await self.emit_event(child_event)
+
+    async def emit_dns_children_raw(self, event):
+        for rdtype, answers in event.raw_dns_records.items():
+            if self.emit_raw_records and rdtype not in ("A", "AAAA", "CNAME", "PTR"):
+                for answer in answers:
+                    text_answer = answer.to_text()
+                    child_hash = hash(f"{event.host}:{rdtype}:{text_answer}")
+                    if child_hash not in self.children_emitted_raw:
+                        self.children_emitted_raw.add(child_hash)
+                        await self.emit_event(
+                            {"host": str(event.host), "type": rdtype, "answer": text_answer},
+                            "RAW_DNS_RECORD",
+                            parent=event,
+                            tags=[f"{rdtype.lower()}-record"],
+                            context=f"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}",
+                        )
 
     def check_scope(self, event):
         whitelisted = False
@@ -124,13 +200,14 @@ def check_scope(self, event):
             # update resolved hosts
             event.resolved_hosts.update(hosts)
             for host in hosts:
-                # having a CNAME to an in-scope resource doesn't make you in-scope
+                # having a CNAME to an in-scope host doesn't make you in-scope
                 if rdtype != "CNAME":
                     if not whitelisted:
                         with suppress(ValidationError):
                             if self.scan.whitelisted(host):
                                 whitelisted = True
                                 event.add_tag(f"dns-whitelisted-{rdtype}")
+                # but a CNAME to a blacklisted host means you're blacklisted
                 if not blacklisted:
                     with suppress(ValidationError):
                         if self.scan.blacklisted(host):
@@ -142,36 +219,43 @@ def check_scope(self, event):
         return whitelisted, blacklisted
 
     async def resolve_event(self, event, types):
-        raw_records = {}
+        if not types:
+            return
         event_host = str(event.host)
         queries = [(event_host, rdtype) for rdtype in types]
         dns_errors = {}
-        async for (query, rdtype), (answer, errors) in self.helpers.dns.resolve_raw_batch(queries):
+        async for (query, rdtype), (answers, errors) in self.helpers.dns.resolve_raw_batch(queries):
+            # errors
             try:
                 dns_errors[rdtype].update(errors)
             except KeyError:
                 dns_errors[rdtype] = set(errors)
-            # raw dnspython objects
-            try:
-                raw_records[rdtype].add(answer)
-            except KeyError:
-                raw_records[rdtype] = {answer}
-            # hosts
-            for _rdtype, host in extract_targets(answer):
-                event.add_tag(f"{_rdtype}-record")
+            for answer in answers:
+                # raw dnspython answers
                 try:
-                    event.dns_children[_rdtype].add(host)
+                    event.raw_dns_records[rdtype].add(answer)
                 except KeyError:
-                    event.dns_children[_rdtype] = {host}
+                    event.raw_dns_records[rdtype] = {answer}
+                # hosts
+                for _rdtype, host in extract_targets(answer):
+                    event.add_tag(f"{_rdtype}-record")
+                    try:
+                        event.dns_children[_rdtype].add(host)
+                    except KeyError:
+                        event.dns_children[_rdtype] = {host}
+                    # check for private IPs
+                    try:
+                        ip = ipaddress.ip_address(host)
+                        if ip.is_private:
+                            event.add_tag("private-ip")
+                    except ValueError:
+                        continue
+
         # tag event with errors
         for rdtype, errors in dns_errors.items():
             # only consider it an error if there weren't any results for that rdtype
             if errors and not rdtype in event.dns_children:
                 event.add_tag(f"{rdtype}-error")
-        return raw_records
-
-    async def handle_wildcard_event(self, event):
-        pass
 
     def get_dns_parent(self, event):
         """
@@ -181,18 +265,24 @@ def get_dns_parent(self, event):
             if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"):
                 blacklisted = any(t.startswith("dns-blacklisted-") for t in parent.tags)
                 whitelisted = any(t.startswith("dns-whitelisted-") for t in parent.tags)
-                return parent, whitelisted, blacklisted, False
+                new_event = parent is event
+                return parent, whitelisted, blacklisted, new_event
         tags = set()
         if "target" in event.tags:
             tags.add("target")
-        return self.scan.make_event(
-            event.host,
-            "DNS_NAME",
-            module=self.host_module,
-            parent=event,
-            context="{event.parent.type} has host {event.type}: {event.host}",
-            tags=tags,
-        ), None, None, True
+        return (
+            self.scan.make_event(
+                event.host,
+                "DNS_NAME",
+                module=self.host_module,
+                parent=event,
+                context="{event.parent.type} has host {event.type}: {event.host}",
+                tags=tags,
+            ),
+            None,
+            None,
+            True,
+        )
 
     @property
     def emit_raw_records(self):
@@ -218,14 +308,3 @@ def _make_dummy_module(self, name):
             dummy_module.suppress_dupes = False
             self.scan.dummy_modules[name] = dummy_module
         return dummy_module
-
-    def _dns_child_dedup_hash(self, parent_host, host, rdtype):
-        # we deduplicate NS records by their parent domain
-        # because otherwise every DNS_NAME has one, and it gets super messy
-        if rdtype == "NS":
-            _, parent_domain = self.helpers.split_domain(parent_host)
-            return hash(f"{parent_domain}:{host}")
-        return hash(f"{parent_host}:{host}:{rdtype}")
-
-    def _main_outgoing_dedup_hash(self, event):
-        return hash(f"{event.host}")
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
index 1c9631facd..86110a6cbd 100644
--- a/bbot/test/bbot_fixtures.py
+++ b/bbot/test/bbot_fixtures.py
@@ -208,9 +208,9 @@ class bbot_events:
     return bbot_events
 
 
-@pytest.fixture(scope="session", autouse=True)
-def install_all_python_deps():
-    deps_pip = set()
-    for module in DEFAULT_PRESET.module_loader.preloaded().values():
-        deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+# @pytest.fixture(scope="session", autouse=True)
+# def install_all_python_deps():
+#     deps_pip = set()
+#     for module in DEFAULT_PRESET.module_loader.preloaded().values():
+#         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
+#     subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index b10fcc5440..6d7a83b66d 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -38,12 +38,14 @@ async def test_dns_engine(bbot_scanner):
     results = [_ async for _ in scan.helpers.resolve_raw_batch((("one.one.one.one", "A"), ("1.1.1.1", "PTR")))]
     pass_1 = False
     pass_2 = False
-    for (query, rdtype), (result, errors) in results:
-        result = extract_targets(result)
-        _results = [r[1] for r in result]
-        if query == "one.one.one.one" and "1.1.1.1" in _results:
+    for (query, rdtype), (answers, errors) in results:
+        results = []
+        for answer in answers:
+            for t in extract_targets(answer):
+                results.append(t[1])
+        if query == "one.one.one.one" and "1.1.1.1" in query:
             pass_1 = True
-        elif query == "1.1.1.1" and "one.one.one.one" in _results:
+        elif query == "1.1.1.1" and "one.one.one.one" in query:
             pass_2 = True
     assert pass_1 and pass_2
 
@@ -115,7 +117,10 @@ async def test_dns_resolution(bbot_scanner):
 
     # custom batch resolution
     batch_results = [r async for r in dnsengine.resolve_raw_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])]
-    batch_results = [(response.to_text(), response.rdtype.name) for query, (response, errors) in batch_results]
+    batch_results = []
+    for query, (answers, errors) in batch_results:
+        for answer in answers:
+            batch_results.append((answer.to_text(), answer.rdtype.name))
     assert len(batch_results) == 3
     assert any(answer == "1.0.0.1" and rdtype == "A" for answer, rdtype in batch_results)
     assert any(answer == "one.one.one.one." and rdtype == "PTR" for answer, rdtype in batch_results)
@@ -148,11 +153,7 @@ async def test_dns_resolution(bbot_scanner):
     resolved_hosts_event1 = scan.make_event("one.one.one.one", "DNS_NAME", parent=scan.root_event)
     resolved_hosts_event2 = scan.make_event("http://one.one.one.one/", "URL_UNVERIFIED", parent=scan.root_event)
     dnsresolve = scan.modules["dnsresolve"]
-    assert hash(resolved_hosts_event1.host) not in dnsresolve._event_cache
-    assert hash(resolved_hosts_event2.host) not in dnsresolve._event_cache
     await dnsresolve.handle_event(resolved_hosts_event1)
-    assert hash(resolved_hosts_event1.host) in dnsresolve._event_cache
-    assert hash(resolved_hosts_event2.host) in dnsresolve._event_cache
     await dnsresolve.handle_event(resolved_hosts_event2)
     assert "1.1.1.1" in resolved_hosts_event2.resolved_hosts
     # URL event should not have dns_children
@@ -160,6 +161,8 @@ async def test_dns_resolution(bbot_scanner):
     assert resolved_hosts_event1.resolved_hosts == resolved_hosts_event2.resolved_hosts
     # DNS_NAME event should have dns_children
     assert "1.1.1.1" in resolved_hosts_event1.dns_children["A"]
+    assert "A" in resolved_hosts_event1.raw_dns_records
+    assert "AAAA" in resolved_hosts_event1.raw_dns_records
     assert "a-record" in resolved_hosts_event1.tags
     assert not "a-record" in resolved_hosts_event2.tags
 

From be70ae388e0b5496e271307799f1ab82ba7362ad Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Thu, 29 Aug 2024 17:22:48 -0400
Subject: [PATCH 03/12] bugfixing

---
 bbot/core/helpers/dns/brute.py           | 14 +++++---------
 bbot/modules/templates/subdomain_enum.py |  6 +++---
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
index 9b5f55f516..d541f9a677 100644
--- a/bbot/core/helpers/dns/brute.py
+++ b/bbot/core/helpers/dns/brute.py
@@ -39,18 +39,14 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
             type = "A"
         type = str(type).strip().upper()
 
-        domain_wildcard_rdtypes = set()
-        for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items():
-            for rdtype, results in rdtypes.items():
-                if results:
-                    domain_wildcard_rdtypes.add(rdtype)
-        if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]):
-            self.log.info(
-                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
+        wildcard_rdtypes = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME"))
+        if wildcard_rdtypes:
+            self.log.hugewarning(
+                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})"
             )
             return []
         else:
-            self.log.debug(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
+            self.log.hugeinfo(f"{domain}: A is not in domain_wildcard_rdtypes:{wildcard_rdtypes}")
 
         canaries = self.gen_random_subdomains(self.num_canaries)
         canaries_list = list(canaries)
diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 28f775d2a3..9b9a72ebe7 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -119,9 +119,9 @@ async def query(self, query, parse_fn=None, request_fn=None):
 
     async def _is_wildcard(self, query):
         if self.helpers.is_dns_name(query):
-            for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query)).items():
-                if any(t in wildcard_rdtypes for t in ("A", "AAAA", "CNAME")):
-                    return True
+            wildcard_rdtypes = await self.helpers.is_wildcard_domain(query, rdtypes=("A", "AAAA", "CNAME"))
+            if wildcard_rdtypes:
+                return True
         return False
 
     async def filter_event(self, event):

From ead400ba9eff0500b5482d0ccc503c0b0a83aecb Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Fri, 30 Aug 2024 17:30:41 -0400
Subject: [PATCH 04/12] more WIP dns rework

---
 bbot/core/event/base.py              |  20 +-
 bbot/core/helpers/dns/dns.py         |  10 +-
 bbot/core/helpers/dns/engine.py      |  59 +++--
 bbot/core/helpers/dns/mock.py        |  33 ++-
 bbot/modules/internal/dnsresolve.py  |  45 ++--
 bbot/modules/internal/speculate.py   |   2 +-
 bbot/test/bbot_fixtures.py           |  12 +-
 bbot/test/test_step_1/test_dns.py    | 339 ++++++++++++++++++++++++---
 bbot/test/test_step_1/test_events.py |  10 +
 9 files changed, 431 insertions(+), 99 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 94df01cf57..fde835636b 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1054,6 +1054,17 @@ def __init__(self, *args, **kwargs):
             if parent_module_type == "DNS":
                 self.dns_resolve_distance += 1
         # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
+        # tag subdomain / domain
+        if is_subdomain(self.host):
+            self.add_tag("subdomain")
+        elif is_domain(self.host):
+            self.add_tag("domain")
+        # tag private IP
+        try:
+            if self.host.is_private:
+                self.add_tag("private-ip")
+        except AttributeError:
+            pass
 
 
 class IP_RANGE(DnsEvent):
@@ -1070,13 +1081,6 @@ def _host(self):
 
 
 class DNS_NAME(DnsEvent):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if is_subdomain(self.data):
-            self.add_tag("subdomain")
-        elif is_domain(self.data):
-            self.add_tag("domain")
-
     def sanitize_data(self, data):
         return validators.validate_host(data)
 
@@ -1499,7 +1503,7 @@ class FILESYSTEM(DictPathEvent):
     pass
 
 
-class RAW_DNS_RECORD(DictHostEvent):
+class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     # don't emit raw DNS records for affiliates
     _always_emit_tags = ["target"]
 
diff --git a/bbot/core/helpers/dns/dns.py b/bbot/core/helpers/dns/dns.py
index 7574740153..43380b7465 100644
--- a/bbot/core/helpers/dns/dns.py
+++ b/bbot/core/helpers/dns/dns.py
@@ -66,7 +66,7 @@ def __init__(self, parent_helper):
         self.resolver.timeout = self.timeout
         self.resolver.lifetime = self.timeout
 
-        self.runaway_limit = self.config.get("runaway_limit", 5)
+        self.runaway_limit = self.dns_config.get("runaway_limit", 5)
 
         # wildcard handling
         self.wildcard_disable = self.dns_config.get("wildcard_disable", False)
@@ -119,7 +119,7 @@ def brute(self):
 
     @async_cachedmethod(
         lambda self: self._is_wildcard_cache,
-        key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes))),
+        key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)),
     )
     async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         """
@@ -194,8 +194,8 @@ def _wildcard_prevalidation(self, host):
 
         return host
 
-    async def _mock_dns(self, mock_data):
+    async def _mock_dns(self, mock_data, custom_lookup_fn=None):
         from .mock import MockResolver
 
-        self.resolver = MockResolver(mock_data)
-        await self.run_and_return("_mock_dns", mock_data=mock_data)
+        self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn)
+        await self.run_and_return("_mock_dns", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn)
diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py
index e72b05b80b..481e6b0760 100644
--- a/bbot/core/helpers/dns/engine.py
+++ b/bbot/core/helpers/dns/engine.py
@@ -400,6 +400,9 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
 
         This can reliably tell the difference between a valid DNS record and a wildcard within a wildcard domain.
 
+        It works by making a bunch of random DNS queries to the parent domain, compiling a list of wildcard IPs,
+        then comparing those to the IPs of the host in question. If the host's IP matches the wildcard ones, it's a wildcard.
+
         If you want to know whether a domain is using wildcard DNS, use `is_wildcard_domain()` instead.
 
         Args:
@@ -413,9 +416,6 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
                 Values are tuples where the first element is a boolean indicating if the query is a wildcard,
                 and the second element is the wildcard parent if it's a wildcard.
 
-        Raises:
-            ValueError: If only one of `ips` or `rdtype` is specified or if no valid IPs are specified.
-
         Examples:
             >>> is_wildcard("www.github.io", rdtypes=["A", "AAAA", "MX"])
             {"A": (True, "github.io"), "AAAA": (True, "github.io"), "MX": (False, "github.io")}
@@ -445,7 +445,7 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
                 else:
                     if errors:
                         self.debug(f"Failed to resolve {query} ({rdtype}) during wildcard detection")
-                        result[rdtype] = (None, query)
+                        result[rdtype] = ("ERROR", query)
 
         # clean + process the raw records into a baseline
         baseline = {}
@@ -470,12 +470,15 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         # once we've resolved the base query and have IP addresses to work with
         # we can compare the IPs to the ones we have on file for wildcards
 
+        # only bother to check the rdypes that actually resolve
+        rdtypes_to_check = set(raw_dns_records)
+
         # for every parent domain, starting with the shortest
         parents = list(domain_parents(query))
         for parent in parents[::-1]:
 
             # check if the parent domain is set up with wildcards
-            wildcard_results = await self.is_wildcard_domain(parent, rdtypes)
+            wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check)
 
             # for every rdtype
             for rdtype in list(baseline_raw):
@@ -487,18 +490,26 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
                 _baseline = baseline.get(rdtype, set())
                 _baseline_raw = baseline_raw.get(rdtype, set())
 
-                wildcards = wildcard_results.get(rdtype, None)
+                wildcard_rdtypes = wildcard_results.get(parent, {})
+                wildcards = wildcard_rdtypes.get(rdtype, None)
                 if wildcards is None:
                     continue
                 wildcards, wildcard_raw = wildcards
 
-                # check if any of our baseline IPs are in the wildcard results
-                is_wildcard = any(r in wildcards for r in _baseline)
-                is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw)
+                if wildcard_raw:
+                    # skip this rdtype from now on
+                    rdtypes_to_check.remove(rdtype)
 
-                # if there are any matches, we have a wildcard
-                if is_wildcard or is_wildcard_raw:
-                    result[rdtype] = (True, query)
+                    # check if any of our baseline IPs are in the wildcard results
+                    is_wildcard = any(r in wildcards for r in _baseline)
+                    is_wildcard_raw = any(r in wildcard_raw for r in _baseline_raw)
+
+                    # if there are any matches, we have a wildcard
+                    if is_wildcard or is_wildcard_raw:
+                        result[rdtype] = (True, parent)
+                    else:
+                        # otherwise, it's still suspicious, because we had random stuff resolve at this level
+                        result[rdtype] = ("POSSIBLE", parent)
 
         # any rdtype that wasn't a wildcard, mark it as False
         for rdtype, answers in baseline_raw.items():
@@ -531,11 +542,13 @@ async def is_wildcard_domain(self, domain, rdtypes):
         if isinstance(rdtypes, str):
             rdtypes = [rdtypes]
         rdtypes = set(rdtypes)
+
         wildcard_results = {}
         # make a list of its parents
         parents = list(domain_parents(domain, include_self=True))
         # and check each of them, beginning with the highest parent (i.e. the root domain)
         for i, host in enumerate(parents[::-1]):
+            host_results = {}
             queries = [((host, rdtype), {}) for rdtype in rdtypes]
             async for ((_, rdtype), _, _), (results, results_raw) in self.task_pool(
                 self._is_wildcard_zone, args_kwargs=queries
@@ -543,13 +556,10 @@ async def is_wildcard_domain(self, domain, rdtypes):
                 # if we hit a wildcard, we can skip this rdtype from now on
                 if results_raw:
                     rdtypes.remove(rdtype)
-                    wildcard_results[rdtype] = results, results_raw
+                    host_results[rdtype] = results, results_raw
 
-        if wildcard_results:
-            wildcard_rdtypes_str = ",".join(sorted(wildcard_results))
-            self.log.info(f"Encountered domain with wildcard DNS ({wildcard_rdtypes_str}): {host}")
-        else:
-            self.log.verbose(f"Finished checking {host}, it is not a wildcard")
+            if host_results:
+                wildcard_results[host] = host_results
 
         return wildcard_results
 
@@ -587,7 +597,7 @@ async def _is_wildcard_zone(self, host, rdtype):
                             wildcard_results.add(t)
 
                 if wildcard_results:
-                    self.debug(f"Finished checking {host}:{rdtype}, it is a wildcard")
+                    self.log.info(f"Encountered domain with wildcard DNS ({rdtype}): *.{host}")
                 else:
                     self.debug(f"Finished checking {host}:{rdtype}, it is not a wildcard")
                 self._wildcard_cache[host_hash] = wildcard_results, wildcard_results_raw
@@ -643,7 +653,14 @@ def debug(self, *args, **kwargs):
     def in_tests(self):
         return os.getenv("BBOT_TESTING", "") == "True"
 
-    async def _mock_dns(self, mock_data):
+    async def _mock_dns(self, mock_data, custom_lookup_fn=None):
         from .mock import MockResolver
 
-        self.resolver = MockResolver(mock_data)
+        def deserialize_function(func_source):
+            assert self.in_tests, "Can only mock when BBOT_TESTING=True"
+            if func_source is None:
+                return None
+            exec(func_source)
+            return locals()["custom_lookup"]
+
+        self.resolver = MockResolver(mock_data, custom_lookup_fn=deserialize_function(custom_lookup_fn))
diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py
index 70d978affe..02fdbaa638 100644
--- a/bbot/core/helpers/dns/mock.py
+++ b/bbot/core/helpers/dns/mock.py
@@ -1,10 +1,14 @@
 import dns
+import logging
+
+log = logging.getLogger("bbot.core.helpers.dns.mock")
 
 
 class MockResolver:
 
-    def __init__(self, mock_data=None):
+    def __init__(self, mock_data=None, custom_lookup_fn=None):
         self.mock_data = mock_data if mock_data else {}
+        self._custom_lookup_fn = custom_lookup_fn
         self.nameservers = ["127.0.0.1"]
 
     async def resolve_address(self, ipaddr, *args, **kwargs):
@@ -13,12 +17,22 @@ async def resolve_address(self, ipaddr, *args, **kwargs):
         modified_kwargs["rdtype"] = "PTR"
         return await self.resolve(str(dns.reversename.from_address(ipaddr)), *args, **modified_kwargs)
 
-    def create_dns_response(self, query_name, rdtype):
-        query_name = query_name.strip(".")
-        answers = self.mock_data.get(query_name, {}).get(rdtype, [])
-        if not answers:
-            raise dns.resolver.NXDOMAIN(f"No answer found for {query_name} {rdtype}")
+    def _lookup(self, query, rdtype):
+        query = query.strip(".")
+        ret = []
+        if self._custom_lookup_fn is not None:
+            answers = self._custom_lookup_fn(query, rdtype)
+            if answers is not None:
+                ret.extend(list(answers))
+        answers = self.mock_data.get(query, {}).get(rdtype, [])
+        if answers:
+            ret.extend(list(answers))
+        if not ret:
+            raise dns.resolver.NXDOMAIN(f"No answer found for {query} {rdtype}")
+        return ret
 
+    def create_dns_response(self, query_name, answers, rdtype):
+        query_name = query_name.strip(".")
         message_text = f"""id 1234
 opcode QUERY
 rcode NOERROR
@@ -27,10 +41,13 @@ def create_dns_response(self, query_name, rdtype):
 {query_name}. IN {rdtype}
 ;ANSWER"""
         for answer in answers:
+            if answer == "":
+                answer = '""'
             message_text += f"\n{query_name}. 1 IN {rdtype} {answer}"
 
         message_text += "\n;AUTHORITY\n;ADDITIONAL\n"
         message = dns.message.from_text(message_text)
+        log.verbose(message_text)
         return message
 
     async def resolve(self, query_name, rdtype=None):
@@ -49,7 +66,9 @@ async def resolve(self, query_name, rdtype=None):
             raise dns.resolver.NXDOMAIN
 
         try:
-            response = self.create_dns_response(query_name, rdtype)
+            answers = self._lookup(query_name, rdtype)
+            log.verbose(f"Answers for {query_name}:{rdtype}: {answers}")
+            response = self.create_dns_response(query_name, answers, rdtype)
             answer = dns.resolver.Answer(domain_name, rdtype_obj, dns.rdataclass.IN, response)
             return answer
         except dns.resolver.NXDOMAIN:
diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index 8ffbeb60ba..b73a49fba1 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -41,6 +41,7 @@ async def setup(self):
         self.host_module = self.HostModule(self.scan)
         self.children_emitted = set()
         self.children_emitted_raw = set()
+        self.hosts_resolved = set()
 
         return True
 
@@ -60,6 +61,7 @@ async def handle_event(self, event, **kwargs):
 
         # first, we find or create the main DNS_NAME or IP_ADDRESS associated with this event
         main_host_event, whitelisted, blacklisted, new_event = self.get_dns_parent(event)
+        original_tags = set(event.tags)
 
         # minimal resolution - first, we resolve A/AAAA records for scope purposes
         if new_event or event is main_host_event:
@@ -79,33 +81,37 @@ async def handle_event(self, event, **kwargs):
             if main_host_event.scope_distance < self._dns_search_distance:
                 await self.resolve_event(main_host_event, types=non_minimal_rdtypes)
                 # check for wildcards if we're within the scan's search distance
-                if (
-                    new_event
-                    and main_host_event.scope_distance <= self.scan.scope_search_distance
-                    and not "domain" in main_host_event.tags
-                ):
+                if new_event and main_host_event.scope_distance <= self.scan.scope_search_distance:
                     await self.handle_wildcard_event(main_host_event)
 
+        # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}")
+
+        dns_tags = main_host_event.tags.difference(original_tags)
+
         dns_resolve_distance = getattr(main_host_event, "dns_resolve_distance", 0)
         runaway_dns = dns_resolve_distance >= self.helpers.dns.runaway_limit
         if runaway_dns:
             # kill runaway DNS chains
-            # TODO: test this
             self.debug(
                 f"Skipping DNS children for {event} because their DNS resolve distances would be greater than the configured value for this scan ({self.helpers.dns.runaway_limit})"
             )
+            main_host_event.add_tag(f"runaway-dns-{dns_resolve_distance}")
         else:
             # emit dns children
             await self.emit_dns_children(main_host_event)
-            await self.emit_dns_children_raw(main_host_event)
+            await self.emit_dns_children_raw(main_host_event, dns_tags)
 
-        # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
-        if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags:
-            main_host_event.type = "DNS_NAME_UNRESOLVED"
+            # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
+            if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags:
+                main_host_event.type = "DNS_NAME_UNRESOLVED"
 
-        # emit the main DNS_NAME or IP_ADDRESS
-        if new_event and event is not main_host_event and main_host_event.scope_distance <= self._dns_search_distance:
-            await self.emit_event(main_host_event)
+            # emit the main DNS_NAME or IP_ADDRESS
+            if (
+                new_event
+                and event is not main_host_event
+                and main_host_event.scope_distance <= self._dns_search_distance
+            ):
+                await self.emit_event(main_host_event)
 
         # transfer scope distance to event
         event.scope_distance = main_host_event.scope_distance
@@ -122,8 +128,9 @@ async def handle_wildcard_event(self, event):
             elif is_wildcard == True:
                 event.add_tag("wildcard")
                 wildcard_tag = "wildcard"
-            elif is_wildcard == None:
-                wildcard_tag = "error"
+            else:
+                event.add_tag(f"wildcard-{is_wildcard}")
+                wildcard_tag = f"wildcard-{is_wildcard}"
             event.add_tag(f"{rdtype}-{wildcard_tag}")
 
         # wildcard event modification (www.evilcorp.com --> _wildcard.evilcorp.com)
@@ -175,8 +182,10 @@ async def emit_dns_children(self, event):
                         self.debug(f"Queueing DNS child for {event}: {child_event}")
                         await self.emit_event(child_event)
 
-    async def emit_dns_children_raw(self, event):
+    async def emit_dns_children_raw(self, event, dns_tags):
         for rdtype, answers in event.raw_dns_records.items():
+            rdtype_lower = rdtype.lower()
+            tags = {t for t in dns_tags if rdtype_lower in t.split("-")}
             if self.emit_raw_records and rdtype not in ("A", "AAAA", "CNAME", "PTR"):
                 for answer in answers:
                     text_answer = answer.to_text()
@@ -187,7 +196,7 @@ async def emit_dns_children_raw(self, event):
                             {"host": str(event.host), "type": rdtype, "answer": text_answer},
                             "RAW_DNS_RECORD",
                             parent=event,
-                            tags=[f"{rdtype.lower()}-record"],
+                            tags=tags,
                             context=f"{rdtype} lookup on {{event.parent.host}} produced {{event.type}}",
                         )
 
@@ -231,6 +240,7 @@ async def resolve_event(self, event, types):
             except KeyError:
                 dns_errors[rdtype] = set(errors)
             for answer in answers:
+                event.add_tag(f"{rdtype}-record")
                 # raw dnspython answers
                 try:
                     event.raw_dns_records[rdtype].add(answer)
@@ -238,7 +248,6 @@ async def resolve_event(self, event, types):
                     event.raw_dns_records[rdtype] = {answer}
                 # hosts
                 for _rdtype, host in extract_targets(answer):
-                    event.add_tag(f"{_rdtype}-record")
                     try:
                         event.dns_children[_rdtype].add(host)
                     except KeyError:
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index bb73094ff0..622c0bfe0e 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -91,7 +91,7 @@ async def handle_event(self, event):
 
         # parent domains
         if event.type.startswith("DNS_NAME"):
-            parent = self.helpers.parent_domain(event.data)
+            parent = self.helpers.parent_domain(event.host_original)
             if parent != event.data:
                 await self.emit_event(
                     parent, "DNS_NAME", parent=event, context=f"speculated parent {{event.type}}: {{event.data}}"
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
index 86110a6cbd..1c9631facd 100644
--- a/bbot/test/bbot_fixtures.py
+++ b/bbot/test/bbot_fixtures.py
@@ -208,9 +208,9 @@ class bbot_events:
     return bbot_events
 
 
-# @pytest.fixture(scope="session", autouse=True)
-# def install_all_python_deps():
-#     deps_pip = set()
-#     for module in DEFAULT_PRESET.module_loader.preloaded().values():
-#         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-#     subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+@pytest.fixture(scope="session", autouse=True)
+def install_all_python_deps():
+    deps_pip = set()
+    for module in DEFAULT_PRESET.module_loader.preloaded().values():
+        deps_pip.update(set(module.get("deps", {}).get("pip", [])))
+    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 6d7a83b66d..517e42dc6b 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -43,9 +43,9 @@ async def test_dns_engine(bbot_scanner):
         for answer in answers:
             for t in extract_targets(answer):
                 results.append(t[1])
-        if query == "one.one.one.one" and "1.1.1.1" in query:
+        if query == "one.one.one.one" and "1.1.1.1" in results:
             pass_1 = True
-        elif query == "1.1.1.1" and "one.one.one.one" in query:
+        elif query == "1.1.1.1" and "one.one.one.one" in results:
             pass_2 = True
     assert pass_1 and pass_2
 
@@ -117,13 +117,13 @@ async def test_dns_resolution(bbot_scanner):
 
     # custom batch resolution
     batch_results = [r async for r in dnsengine.resolve_raw_batch([("1.1.1.1", "PTR"), ("one.one.one.one", "A")])]
-    batch_results = []
+    batch_results_new = []
     for query, (answers, errors) in batch_results:
         for answer in answers:
-            batch_results.append((answer.to_text(), answer.rdtype.name))
-    assert len(batch_results) == 3
-    assert any(answer == "1.0.0.1" and rdtype == "A" for answer, rdtype in batch_results)
-    assert any(answer == "one.one.one.one." and rdtype == "PTR" for answer, rdtype in batch_results)
+            batch_results_new.append((answer.to_text(), answer.rdtype.name))
+    assert len(batch_results_new) == 3
+    assert any(answer == "1.0.0.1" and rdtype == "A" for answer, rdtype in batch_results_new)
+    assert any(answer == "one.one.one.one." and rdtype == "PTR" for answer, rdtype in batch_results_new)
 
     # dns cache
     dnsengine._dns_cache.clear()
@@ -184,40 +184,316 @@ async def test_dns_resolution(bbot_scanner):
 
 @pytest.mark.asyncio
 async def test_wildcards(bbot_scanner):
+
     scan = bbot_scanner("1.1.1.1")
     helpers = scan.helpers
 
-    from bbot.core.helpers.dns.engine import DNSEngine
+    from bbot.core.helpers.dns.engine import DNSEngine, all_rdtypes
 
-    dnsengine = DNSEngine(None)
+    dnsengine = DNSEngine(None, debug=True)
 
-    # wildcards
-    wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io")
-    assert hash("github.io") in dnsengine._wildcard_cache
-    assert hash("asdf.github.io") in dnsengine._wildcard_cache
+    # is_wildcard_domain
+    wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io", all_rdtypes)
+    assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2)
+    for rdtype in all_rdtypes:
+        assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache
+        if not rdtype in ("A", "AAAA"):
+            assert hash(f"asdf.github.io:{rdtype}") in dnsengine._wildcard_cache
     assert "github.io" in wildcard_domains
     assert "A" in wildcard_domains["github.io"]
     assert "SRV" not in wildcard_domains["github.io"]
-    assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"])
+    assert wildcard_domains["github.io"]["A"] and all(helpers.is_ip(r) for r in wildcard_domains["github.io"]["A"][0])
     dnsengine._wildcard_cache.clear()
 
-    wildcard_rdtypes = await dnsengine.is_wildcard("blacklanternsecurity.github.io")
-    assert "A" in wildcard_rdtypes
-    assert "SRV" not in wildcard_rdtypes
-    assert wildcard_rdtypes["A"] == (True, "github.io")
-    assert hash("github.io") in dnsengine._wildcard_cache
-    assert len(dnsengine._wildcard_cache[hash("github.io")]) > 0
-    dnsengine._wildcard_cache.clear()
+    # is_wildcard
+    for test_domain in ("blacklanternsecurity.github.io", "asdf.asdf.asdf.github.io"):
+        wildcard_rdtypes = await dnsengine.is_wildcard(test_domain, all_rdtypes)
+        assert "A" in wildcard_rdtypes
+        assert "SRV" not in wildcard_rdtypes
+        assert wildcard_rdtypes["A"] == (True, "github.io")
+        assert wildcard_rdtypes["AAAA"] == (True, "github.io")
+        assert len(dnsengine._wildcard_cache) == 2
+        for rdtype in ("A", "AAAA"):
+            assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache
+            assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")]) == 2
+            assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][0]) > 0
+            assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][1]) > 0
+        dnsengine._wildcard_cache.clear()
+
+    ### wildcard TXT record ###
+
+    custom_lookup = """
+def custom_lookup(query, rdtype):
+    if rdtype == "TXT" and query.strip(".").endswith("test.evilcorp.com"):
+        return {""}
+"""
+
+    mock_data = {
+        "evilcorp.com": {"A": ["127.0.0.1"]},
+        "test.evilcorp.com": {"A": ["127.0.0.2"]},
+        "www.test.evilcorp.com": {"AAAA": ["dead::beef"]},
+    }
+
+    # basic sanity checks
+
+    await dnsengine._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
+
+    a_result = await dnsengine.resolve("evilcorp.com")
+    assert a_result == {"127.0.0.1"}
+    aaaa_result = await dnsengine.resolve("www.test.evilcorp.com", type="AAAA")
+    assert aaaa_result == {"dead::beef"}
+    txt_result = await dnsengine.resolve("asdf.www.test.evilcorp.com", type="TXT")
+    assert txt_result == set()
+    txt_result_raw, errors = await dnsengine.resolve_raw("asdf.www.test.evilcorp.com", type="TXT")
+    txt_result_raw = list(txt_result_raw)
+    assert txt_result_raw
+
+    await dnsengine._shutdown()
+
+    # first, we check with wildcard detection disabled
+
+    scan = bbot_scanner(
+        "bbot.fdsa.www.test.evilcorp.com",
+        whitelist=["evilcorp.com"],
+        config={
+            "dns": {"minimal": False, "disable": False, "search_distance": 5, "wildcard_ignore": ["evilcorp.com"]},
+            "speculate": True,
+        },
+    )
+    await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
 
-    wildcard_rdtypes = await dnsengine.is_wildcard("asdf.asdf.asdf.github.io")
-    assert "A" in wildcard_rdtypes
-    assert "SRV" not in wildcard_rdtypes
-    assert wildcard_rdtypes["A"] == (True, "github.io")
-    assert hash("github.io") in dnsengine._wildcard_cache
-    assert not hash("asdf.github.io") in dnsengine._wildcard_cache
-    assert not hash("asdf.asdf.github.io") in dnsengine._wildcard_cache
-    assert not hash("asdf.asdf.asdf.github.io") in dnsengine._wildcard_cache
-    assert len(dnsengine._wildcard_cache[hash("github.io")]) > 0
+    events = [e async for e in scan.async_start()]
+    assert len(events) == 11
+    assert len([e for e in events if e.type == "DNS_NAME"]) == 5
+    assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
+    assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
+        "bbot.fdsa.www.test.evilcorp.com",
+        "evilcorp.com",
+        "fdsa.www.test.evilcorp.com",
+        "test.evilcorp.com",
+        "www.test.evilcorp.com",
+    ]
+
+    dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"}
+    assert dns_names_by_host["evilcorp.com"].tags == {"domain", "private-ip", "in-scope", "a-record"}
+    assert dns_names_by_host["evilcorp.com"].resolved_hosts == {"127.0.0.1"}
+    assert dns_names_by_host["test.evilcorp.com"].tags == {
+        "subdomain",
+        "private-ip",
+        "in-scope",
+        "a-record",
+        "txt-record",
+    }
+    assert dns_names_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"}
+    assert dns_names_by_host["www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "aaaa-record", "txt-record"}
+    assert dns_names_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"}
+    assert dns_names_by_host["fdsa.www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"}
+    assert dns_names_by_host["fdsa.www.test.evilcorp.com"].resolved_hosts == set()
+    assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == {
+        "target",
+        "subdomain",
+        "in-scope",
+        "txt-record",
+    }
+    assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set()
+
+    raw_records_by_host = {e.host: e for e in events if e.type == "RAW_DNS_RECORD"}
+    assert raw_records_by_host["test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"}
+    assert raw_records_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"}
+    assert raw_records_by_host["www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"}
+    assert raw_records_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"}
+    assert raw_records_by_host["fdsa.www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"}
+    assert raw_records_by_host["fdsa.www.test.evilcorp.com"].resolved_hosts == set()
+    assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"}
+    assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set()
+
+    # then we run it again with wildcard detection enabled
+
+    scan = bbot_scanner(
+        "bbot.fdsa.www.test.evilcorp.com",
+        whitelist=["evilcorp.com"],
+        config={
+            "dns": {"minimal": False, "disable": False, "search_distance": 5, "wildcard_ignore": []},
+            "speculate": True,
+        },
+    )
+    await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
+
+    events = [e async for e in scan.async_start()]
+    assert len(events) == 11
+    assert len([e for e in events if e.type == "DNS_NAME"]) == 5
+    assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
+    assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
+        "_wildcard.test.evilcorp.com",
+        "bbot.fdsa.www.test.evilcorp.com",
+        "evilcorp.com",
+        "test.evilcorp.com",
+        "www.test.evilcorp.com",
+    ]
+
+    dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"}
+    assert dns_names_by_host["evilcorp.com"].tags == {"domain", "private-ip", "in-scope", "a-record"}
+    assert dns_names_by_host["evilcorp.com"].resolved_hosts == {"127.0.0.1"}
+    assert dns_names_by_host["test.evilcorp.com"].tags == {
+        "subdomain",
+        "private-ip",
+        "in-scope",
+        "a-record",
+        "txt-record",
+    }
+    assert dns_names_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"}
+    assert dns_names_by_host["_wildcard.test.evilcorp.com"].tags == {
+        "subdomain",
+        "in-scope",
+        "txt-record",
+        "txt-wildcard",
+        "wildcard",
+    }
+    assert dns_names_by_host["_wildcard.test.evilcorp.com"].resolved_hosts == set()
+    assert dns_names_by_host["www.test.evilcorp.com"].tags == {
+        "subdomain",
+        "in-scope",
+        "aaaa-record",
+        "txt-record",
+        "txt-wildcard",
+        "wildcard",
+    }
+    assert dns_names_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"}
+    assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == {
+        "target",
+        "subdomain",
+        "in-scope",
+        "txt-record",
+        "txt-wildcard",
+        "wildcard",
+    }
+    assert dns_names_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set()
+
+    raw_records_by_host = {e.host: e for e in events if e.type == "RAW_DNS_RECORD"}
+    assert raw_records_by_host["test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record"}
+    assert raw_records_by_host["test.evilcorp.com"].resolved_hosts == {"127.0.0.2"}
+    assert raw_records_by_host["www.test.evilcorp.com"].tags == {"subdomain", "in-scope", "txt-record", "txt-wildcard"}
+    assert raw_records_by_host["www.test.evilcorp.com"].resolved_hosts == {"dead::beef"}
+    assert raw_records_by_host["_wildcard.test.evilcorp.com"].tags == {
+        "subdomain",
+        "in-scope",
+        "txt-record",
+        "txt-wildcard",
+    }
+    assert raw_records_by_host["_wildcard.test.evilcorp.com"].resolved_hosts == set()
+    assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].tags == {
+        "subdomain",
+        "in-scope",
+        "txt-record",
+        "txt-wildcard",
+    }
+    assert raw_records_by_host["bbot.fdsa.www.test.evilcorp.com"].resolved_hosts == set()
+
+    ### runaway SRV wildcard ###
+
+    custom_lookup = """
+def custom_lookup(query, rdtype):
+    if rdtype == "SRV" and query.strip(".").endswith("evilcorp.com"):
+        return {f"0 100 389 test.{query}"}
+"""
+
+    mock_data = {
+        "evilcorp.com": {"A": ["127.0.0.1"]},
+        "test.evilcorp.com": {"AAAA": ["dead::beef"]},
+    }
+
+    scan = bbot_scanner(
+        "evilcorp.com",
+        config={
+            "dns": {
+                "minimal": False,
+                "disable": False,
+                "search_distance": 5,
+                "wildcard_ignore": [],
+                "runaway_limit": 3,
+            },
+        },
+    )
+    await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup)
+
+    events = [e async for e in scan.async_start()]
+
+    assert len(events) == 10
+    assert len([e for e in events if e.type == "DNS_NAME"]) == 5
+    assert len([e for e in events if e.type == "RAW_DNS_RECORD"]) == 4
+    assert sorted([e.data for e in events if e.type == "DNS_NAME"]) == [
+        "evilcorp.com",
+        "test.evilcorp.com",
+        "test.test.evilcorp.com",
+        "test.test.test.evilcorp.com",
+        "test.test.test.test.evilcorp.com",
+    ]
+
+    dns_names_by_host = {e.host: e for e in events if e.type == "DNS_NAME"}
+    assert dns_names_by_host["evilcorp.com"].tags == {
+        "target",
+        "a-record",
+        "in-scope",
+        "domain",
+        "srv-record",
+        "private-ip",
+    }
+    assert dns_names_by_host["test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "aaaa-record",
+        "srv-wildcard-possible",
+        "wildcard-possible",
+        "subdomain",
+    }
+    assert dns_names_by_host["test.test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "srv-wildcard-possible",
+        "wildcard-possible",
+        "subdomain",
+    }
+    assert dns_names_by_host["test.test.test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "srv-wildcard-possible",
+        "wildcard-possible",
+        "subdomain",
+    }
+    assert dns_names_by_host["test.test.test.test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "srv-wildcard-possible",
+        "wildcard-possible",
+        "subdomain",
+        "runaway-dns-3",
+    }
+
+    raw_records_by_host = {e.host: e for e in events if e.type == "RAW_DNS_RECORD"}
+    assert raw_records_by_host["evilcorp.com"].tags == {"in-scope", "srv-record", "domain"}
+    assert raw_records_by_host["test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "srv-wildcard-possible",
+        "subdomain",
+    }
+    assert raw_records_by_host["test.test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "srv-wildcard-possible",
+        "subdomain",
+    }
+    assert raw_records_by_host["test.test.test.evilcorp.com"].tags == {
+        "in-scope",
+        "srv-record",
+        "srv-wildcard-possible",
+        "subdomain",
+    }
+
+    scan = bbot_scanner("1.1.1.1")
+    helpers = scan.helpers
+
+    # event resolution
     wildcard_event1 = scan.make_event("wat.asdf.fdsa.github.io", "DNS_NAME", parent=scan.root_event)
     wildcard_event1.scope_distance = 0
     wildcard_event2 = scan.make_event("wats.asd.fdsa.github.io", "DNS_NAME", parent=scan.root_event)
@@ -225,9 +501,6 @@ async def test_wildcards(bbot_scanner):
     wildcard_event3 = scan.make_event("github.io", "DNS_NAME", parent=scan.root_event)
     wildcard_event3.scope_distance = 0
 
-    await dnsengine._shutdown()
-
-    # event resolution
     await scan._prep()
     dnsresolve = scan.modules["dnsresolve"]
     await dnsresolve.handle_event(wildcard_event1)
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 913035d66f..94446e71cc 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -311,6 +311,16 @@ async def test_events(events, helpers):
             {"host": "evilcorp.com", "severity": "WACK", "description": "asdf"}, "VULNERABILITY", dummy=True
         )
 
+    # test tagging
+    ip_event_1 = scan.make_event("8.8.8.8", dummy=True)
+    assert "private-ip" not in ip_event_1.tags
+    ip_event_2 = scan.make_event("192.168.0.1", dummy=True)
+    assert "private-ip" in ip_event_2.tags
+    dns_event_1 = scan.make_event("evilcorp.com", dummy=True)
+    assert "domain" in dns_event_1.tags
+    dns_event_2 = scan.make_event("www.evilcorp.com", dummy=True)
+    assert "subdomain" in dns_event_2.tags
+
     # punycode - event type detection
 
     # japanese

From 5047debe579ee348a94945c56b066701e2d13a2a Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Sat, 31 Aug 2024 18:35:11 -0400
Subject: [PATCH 05/12] fix tests

---
 bbot/core/helpers/dns/engine.py               |  8 ++--
 bbot/core/helpers/dns/mock.py                 |  2 +-
 bbot/modules/internal/dnsresolve.py           |  8 +++-
 bbot/test/test_step_1/test_dns.py             | 34 ++++++++---------
 .../test_manager_scope_accuracy.py            | 38 +++++++++++--------
 5 files changed, 51 insertions(+), 39 deletions(-)

diff --git a/bbot/core/helpers/dns/engine.py b/bbot/core/helpers/dns/engine.py
index 481e6b0760..219339c308 100644
--- a/bbot/core/helpers/dns/engine.py
+++ b/bbot/core/helpers/dns/engine.py
@@ -206,8 +206,8 @@ async def _resolve_hostname(self, query, **kwargs):
         retries = kwargs.pop("retries", self.retries)
         use_cache = kwargs.pop("use_cache", True)
         tries_left = int(retries) + 1
-        parent_hash = hash(f"{parent}:{rdtype}")
-        dns_cache_hash = hash(f"{query}:{rdtype}")
+        parent_hash = hash((parent, rdtype))
+        dns_cache_hash = hash((query, rdtype))
         while tries_left > 0:
             try:
                 if use_cache:
@@ -295,7 +295,7 @@ async def _resolve_ip(self, query, **kwargs):
         tries_left = int(retries) + 1
         results = []
         errors = []
-        dns_cache_hash = hash(f"{query}:PTR")
+        dns_cache_hash = hash((query, "PTR"))
         while tries_left > 0:
             try:
                 if use_cache:
@@ -570,7 +570,7 @@ async def _is_wildcard_zone(self, host, rdtype):
         rdtype = rdtype.upper()
 
         # have we checked this host before?
-        host_hash = hash(f"{host}:{rdtype}")
+        host_hash = hash((host, rdtype))
         async with self._wildcard_lock.lock(host_hash):
             # if we've seen this host before
             try:
diff --git a/bbot/core/helpers/dns/mock.py b/bbot/core/helpers/dns/mock.py
index 02fdbaa638..17ee2759ae 100644
--- a/bbot/core/helpers/dns/mock.py
+++ b/bbot/core/helpers/dns/mock.py
@@ -47,7 +47,7 @@ def create_dns_response(self, query_name, answers, rdtype):
 
         message_text += "\n;AUTHORITY\n;ADDITIONAL\n"
         message = dns.message.from_text(message_text)
-        log.verbose(message_text)
+        # log.verbose(message_text)
         return message
 
     async def resolve(self, query_name, rdtype=None):
diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index b73a49fba1..b1c10ab2f4 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -77,13 +77,17 @@ async def handle_event(self, event, **kwargs):
             return False, "it has a blacklisted DNS record"
 
         if not event_is_ip:
-            # if we're within our dns search distance, resolve the rest of our records
+            # if the event is within our dns search distance, resolve the rest of our records
             if main_host_event.scope_distance < self._dns_search_distance:
                 await self.resolve_event(main_host_event, types=non_minimal_rdtypes)
-                # check for wildcards if we're within the scan's search distance
+                # check for wildcards if the event is within the scan's search distance
                 if new_event and main_host_event.scope_distance <= self.scan.scope_search_distance:
                     await self.handle_wildcard_event(main_host_event)
 
+        # if there weren't any DNS children and it's not an IP address, tag as unresolved
+        if not main_host_event.raw_dns_records and not event_is_ip:
+            main_host_event.add_tag("unresolved")
+
         # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}")
 
         dns_tags = main_host_event.tags.difference(original_tags)
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 517e42dc6b..648c5445b4 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -127,24 +127,24 @@ async def test_dns_resolution(bbot_scanner):
 
     # dns cache
     dnsengine._dns_cache.clear()
-    assert hash(f"1.1.1.1:PTR") not in dnsengine._dns_cache
-    assert hash(f"one.one.one.one:A") not in dnsengine._dns_cache
-    assert hash(f"one.one.one.one:AAAA") not in dnsengine._dns_cache
+    assert hash(("1.1.1.1", "PTR")) not in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "A")) not in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "AAAA")) not in dnsengine._dns_cache
     await dnsengine.resolve("1.1.1.1", use_cache=False)
     await dnsengine.resolve("one.one.one.one", use_cache=False)
-    assert hash(f"1.1.1.1:PTR") not in dnsengine._dns_cache
-    assert hash(f"one.one.one.one:A") not in dnsengine._dns_cache
-    assert hash(f"one.one.one.one:AAAA") not in dnsengine._dns_cache
+    assert hash(("1.1.1.1", "PTR")) not in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "A")) not in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "AAAA")) not in dnsengine._dns_cache
 
     await dnsengine.resolve("1.1.1.1")
-    assert hash(f"1.1.1.1:PTR") in dnsengine._dns_cache
+    assert hash(("1.1.1.1", "PTR")) in dnsengine._dns_cache
     await dnsengine.resolve("one.one.one.one", type="A")
-    assert hash(f"one.one.one.one:A") in dnsengine._dns_cache
-    assert not hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "A")) in dnsengine._dns_cache
+    assert not hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache
     dnsengine._dns_cache.clear()
     await dnsengine.resolve("one.one.one.one", type="AAAA")
-    assert hash(f"one.one.one.one:AAAA") in dnsengine._dns_cache
-    assert not hash(f"one.one.one.one:A") in dnsengine._dns_cache
+    assert hash(("one.one.one.one", "AAAA")) in dnsengine._dns_cache
+    assert not hash(("one.one.one.one", "A")) in dnsengine._dns_cache
 
     await dnsengine._shutdown()
 
@@ -196,9 +196,9 @@ async def test_wildcards(bbot_scanner):
     wildcard_domains = await dnsengine.is_wildcard_domain("asdf.github.io", all_rdtypes)
     assert len(dnsengine._wildcard_cache) == len(all_rdtypes) + (len(all_rdtypes) - 2)
     for rdtype in all_rdtypes:
-        assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache
+        assert hash(("github.io", rdtype)) in dnsengine._wildcard_cache
         if not rdtype in ("A", "AAAA"):
-            assert hash(f"asdf.github.io:{rdtype}") in dnsengine._wildcard_cache
+            assert hash(("asdf.github.io", rdtype)) in dnsengine._wildcard_cache
     assert "github.io" in wildcard_domains
     assert "A" in wildcard_domains["github.io"]
     assert "SRV" not in wildcard_domains["github.io"]
@@ -214,10 +214,10 @@ async def test_wildcards(bbot_scanner):
         assert wildcard_rdtypes["AAAA"] == (True, "github.io")
         assert len(dnsengine._wildcard_cache) == 2
         for rdtype in ("A", "AAAA"):
-            assert hash(f"github.io:{rdtype}") in dnsengine._wildcard_cache
-            assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")]) == 2
-            assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][0]) > 0
-            assert len(dnsengine._wildcard_cache[hash(f"github.io:{rdtype}")][1]) > 0
+            assert hash(("github.io", rdtype)) in dnsengine._wildcard_cache
+            assert len(dnsengine._wildcard_cache[hash(("github.io", rdtype))]) == 2
+            assert len(dnsengine._wildcard_cache[hash(("github.io", rdtype))][0]) > 0
+            assert len(dnsengine._wildcard_cache[hash(("github.io", rdtype))][1]) > 0
         dnsengine._wildcard_cache.clear()
 
     ### wildcard TXT record ###
diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py
index bef9b13e6d..5af147082d 100644
--- a/bbot/test/test_step_1/test_manager_scope_accuracy.py
+++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py
@@ -383,7 +383,6 @@ def custom_setup(scan):
     events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
         "127.0.0.1/31",
         modules=["httpx"],
-        output_modules=["neo4j"],
         _config={
             "dns": {"minimal": False, "search_distance": 2},
             "scope": {"search_distance": 0, "report_distance": 1},
@@ -499,7 +498,7 @@ def custom_setup(scan):
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
     assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
 
-    assert len(all_events) == 23
+    assert len(all_events) == 22
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
     assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
@@ -519,9 +518,8 @@ def custom_setup(scan):
     assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.99" and e.internal == True and e.scope_distance == 3])
 
-    assert len(all_events_nodups) == 21
+    assert len(all_events_nodups) == 20
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
@@ -541,7 +539,6 @@ def custom_setup(scan):
     assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["url"] == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3])
-    assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.99" and e.internal == True and e.scope_distance == 3])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
         assert len(_graph_output_events) == 7
@@ -700,10 +697,11 @@ def custom_setup(scan):
         _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}},
     )
 
-    assert len(events) == 6
+    assert len(events) == 7
     assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
-    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"])
+    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"])
+    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
     assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
@@ -711,10 +709,11 @@ def custom_setup(scan):
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
     assert 0 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
 
-    assert len(all_events) == 13
+    assert len(all_events) == 14
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
-    assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+    assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 0])
     assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
@@ -736,10 +735,13 @@ def custom_setup(scan):
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
-        assert len(_graph_output_events) == 6
+        assert len(_graph_output_events) == 7
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
-        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
         assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
         assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
         assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
@@ -757,44 +759,50 @@ def custom_setup(scan):
         _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}},
     )
 
-    assert len(events) == 3
+    assert len(events) == 4
     assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
     assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
 
-    assert len(all_events) == 11
+    assert len(all_events) == 13
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2])
     assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2])
     assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
+    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
 
-    assert len(all_events_nodups) == 9
+    assert len(all_events_nodups) == 11
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
     assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+    assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
+    assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"])
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
-        assert len(_graph_output_events) == 5
+        assert len(_graph_output_events) == 6
         assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
         assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
         assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2])
         assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
         assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
         assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+        assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
         assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
         assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
 

From 485d8279c58fb9d7cb30f800e508eb176097d587 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Sat, 31 Aug 2024 18:44:20 -0400
Subject: [PATCH 06/12] remove hugeinfo

---
 bbot/core/helpers/dns/brute.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
index d541f9a677..168815bf62 100644
--- a/bbot/core/helpers/dns/brute.py
+++ b/bbot/core/helpers/dns/brute.py
@@ -45,8 +45,6 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
                 f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})"
             )
             return []
-        else:
-            self.log.hugeinfo(f"{domain}: A is not in domain_wildcard_rdtypes:{wildcard_rdtypes}")
 
         canaries = self.gen_random_subdomains(self.num_canaries)
         canaries_list = list(canaries)

From 910980e4d547b8a6986104bdc4c3606827b03dad Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Sat, 31 Aug 2024 19:12:41 -0400
Subject: [PATCH 07/12] fix stats tests

---
 bbot/test/test_step_1/test_modules_basic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 76c2373dbb..29335482e6 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -440,8 +440,8 @@ async def handle_event(self, event):
     speculate_stats = scan.stats.module_stats["speculate"]
     assert speculate_stats.produced == {"DNS_NAME": 1, "URL_UNVERIFIED": 1, "ORG_STUB": 1}
     assert speculate_stats.produced_total == 3
-    assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 2, "URL_UNVERIFIED": 1}
-    assert speculate_stats.consumed_total == 4
+    assert speculate_stats.consumed == {"URL": 1, "DNS_NAME": 2, "URL_UNVERIFIED": 1, "IP_ADDRESS": 2}
+    assert speculate_stats.consumed_total == 6
 
 
 @pytest.mark.asyncio

From 05e3918263264e6cdd0d33108ab519806c9e72bf Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Sat, 31 Aug 2024 20:48:37 -0400
Subject: [PATCH 08/12] more work on tests

---
 bbot/modules/internal/dnsresolve.py           |  3 +-
 .../test_manager_scope_accuracy.py            | 31 ++++++-------------
 bbot/test/test_step_1/test_modules_basic.py   |  2 +-
 3 files changed, 13 insertions(+), 23 deletions(-)

diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index b1c10ab2f4..f07b833f10 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -102,8 +102,9 @@ async def handle_event(self, event, **kwargs):
             main_host_event.add_tag(f"runaway-dns-{dns_resolve_distance}")
         else:
             # emit dns children
-            await self.emit_dns_children(main_host_event)
             await self.emit_dns_children_raw(main_host_event, dns_tags)
+            if not self.minimal:
+                await self.emit_dns_children(main_host_event)
 
             # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
             if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags:
diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py
index 5af147082d..e6eaac7ac1 100644
--- a/bbot/test/test_step_1/test_manager_scope_accuracy.py
+++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py
@@ -697,11 +697,10 @@ def custom_setup(scan):
         _dns_mock={"www.bbottest.notreal": {"A": ["127.0.1.0"]}, "test.notreal": {"A": ["127.0.0.1"]}},
     )
 
-    assert len(events) == 7
+    assert len(events) == 6
     assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
-    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"])
-    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"])
+    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
     assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
@@ -709,11 +708,10 @@ def custom_setup(scan):
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
     assert 0 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
 
-    assert len(all_events) == 14
+    assert len(all_events) == 13
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
-    assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
+    assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 0])
     assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
@@ -735,13 +733,10 @@ def custom_setup(scan):
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
-        assert len(_graph_output_events) == 7
+        assert len(_graph_output_events) == 6
         assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
-        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "speculate"])
-        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A"])
-        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "speculate"])
-        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
+        assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
         assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
         assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
         assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
@@ -759,50 +754,44 @@ def custom_setup(scan):
         _dns_mock={"www.bbottest.notreal": {"A": ["127.0.0.1"]}, "test.notreal": {"A": ["127.0.1.0"]}},
     )
 
-    assert len(events) == 4
+    assert len(events) == 3
     assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
     assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"])
     assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
-    assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
     assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
 
-    assert len(all_events) == 13
+    assert len(all_events) == 11
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2])
     assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2])
     assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
-    assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"])
     assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
 
-    assert len(all_events_nodups) == 11
+    assert len(all_events_nodups) == 9
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 2])
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
     assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
-    assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
-    assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.1.0:9999" and e.internal == True and e.scope_distance == 0])
     assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 3 and str(e.module) == "sslcert"])
     assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
 
     for _graph_output_events in (graph_output_events, graph_output_batch_events):
-        assert len(_graph_output_events) == 6
+        assert len(_graph_output_events) == 5
         assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 1])
         assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
         assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 2])
         assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
         assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
         assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
-        assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.1.0" and e.internal == False and e.scope_distance == 0 and str(e.module) == "A"])
         assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
         assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
 
diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 29335482e6..10cfa5bb6c 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -365,7 +365,7 @@ async def handle_event(self, event):
 
     scan = bbot_scanner(
         "evilcorp.com",
-        config={"speculate": True},
+        config={"speculate": True, "dns": {"minimal": False}},
         output_modules=["python"],
         force_start=True,
     )

From 0c3587a19e0bf984aac5309e78e161e89d4bf95e Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Sat, 31 Aug 2024 21:43:45 -0400
Subject: [PATCH 09/12] more work on tests

---
 bbot/modules/internal/dnsresolve.py           |  3 ++
 bbot/modules/internal/excavate.py             | 35 +++++++++----------
 .../test_manager_scope_accuracy.py            |  2 --
 bbot/test/test_step_1/test_scope.py           |  4 +--
 4 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py
index f07b833f10..f1f215afda 100644
--- a/bbot/modules/internal/dnsresolve.py
+++ b/bbot/modules/internal/dnsresolve.py
@@ -17,6 +17,9 @@ class HostModule(BaseModule):
         _type = "internal"
 
         def _outgoing_dedup_hash(self, event):
+            # this exists to ensure a second, more interesting host isn't passed up
+            # because its ugly cousin spent its one dedup token before it arrived
+            # by removing those race conditions, this makes for more consistent results
             return hash((event, self.name, event.always_emit))
 
     @property
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 794adae255..edb2a766d5 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -313,29 +313,28 @@ class excavateTestRule(ExcavateRule):
 
     _module_threads = 8
 
-    parameter_blacklist = [
-        "__VIEWSTATE",
-        "__EVENTARGUMENT",
-        "__EVENTVALIDATION",
-        "__EVENTTARGET",
-        "__EVENTARGUMENT",
-        "__VIEWSTATEGENERATOR",
-        "__SCROLLPOSITIONY",
-        "__SCROLLPOSITIONX",
-        "ASP.NET_SessionId",
-        "JSESSIONID",
-        "PHPSESSID",
-    ]
+    parameter_blacklist = set(
+        p.lower()
+        for p in [
+            "__VIEWSTATE",
+            "__EVENTARGUMENT",
+            "__EVENTVALIDATION",
+            "__EVENTTARGET",
+            "__EVENTARGUMENT",
+            "__VIEWSTATEGENERATOR",
+            "__SCROLLPOSITIONY",
+            "__SCROLLPOSITIONX",
+            "ASP.NET_SessionId",
+            "JSESSIONID",
+            "PHPSESSID",
+        ]
+    )
 
     yara_rule_name_regex = re.compile(r"rule\s(\w+)\s{")
     yara_rule_regex = re.compile(r"(?s)((?:rule\s+\w+\s*{[^{}]*(?:{[^{}]*}[^{}]*)*[^{}]*(?:/\S*?}[^/]*?/)*)*})")
 
     def in_bl(self, value):
-        in_bl = False
-        for bl_param in self.parameter_blacklist:
-            if bl_param.lower() == value.lower():
-                in_bl = True
-        return in_bl
+        return value.lower() in self.parameter_blacklist
 
     def url_unparse(self, param_type, parsed_url):
         if param_type == "GETPARAM":
diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py
index e6eaac7ac1..ef254f38ce 100644
--- a/bbot/test/test_step_1/test_manager_scope_accuracy.py
+++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py
@@ -603,8 +603,6 @@ def custom_setup(scan):
     assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888"])
 
     assert len(all_events) == 29
-    for e in all_events:
-        log.critical(e)
     assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0])
     assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
diff --git a/bbot/test/test_step_1/test_scope.py b/bbot/test/test_step_1/test_scope.py
index ebd94333f8..7435b82af7 100644
--- a/bbot/test/test_step_1/test_scope.py
+++ b/bbot/test/test_step_1/test_scope.py
@@ -12,7 +12,7 @@ async def setup_after_prep(self, module_test):
         module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
 
     def check(self, module_test, events):
-        assert len(events) == 5
+        assert len(events) == 6
         assert 1 == len(
             [
                 e
@@ -24,7 +24,7 @@ def check(self, module_test, events):
             ]
         )
         # we have two of these because the host module considers "always_emit" in its outgoing deduplication
-        assert 1 == len(
+        assert 2 == len(
             [
                 e
                 for e in events

From 98b1253d6035683607ec9ac61a66f3443fec0ec0 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Sun, 1 Sep 2024 23:39:52 -0400
Subject: [PATCH 10/12] prevent subdomain enum modules from touching
 wildcard-suspicious domains

---
 bbot/modules/templates/subdomain_enum.py      |   7 +-
 bbot/test/test_step_2/module_tests/base.py    |   4 +-
 .../test_template_subdomain_enum.py           | 135 ++++++++++++++++++
 3 files changed, 141 insertions(+), 5 deletions(-)

diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
index 9b9a72ebe7..4f0608fb9e 100644
--- a/bbot/modules/templates/subdomain_enum.py
+++ b/bbot/modules/templates/subdomain_enum.py
@@ -118,10 +118,11 @@ async def query(self, query, parse_fn=None, request_fn=None):
             self.info(f"Error retrieving results for {query}: {e}", trace=True)
 
     async def _is_wildcard(self, query):
+        rdtypes = ("A", "AAAA", "CNAME")
         if self.helpers.is_dns_name(query):
-            wildcard_rdtypes = await self.helpers.is_wildcard_domain(query, rdtypes=("A", "AAAA", "CNAME"))
-            if wildcard_rdtypes:
-                return True
+            for domain, wildcard_rdtypes in (await self.helpers.is_wildcard_domain(query, rdtypes=rdtypes)).items():
+                if any(t in wildcard_rdtypes for t in rdtypes):
+                    return True
         return False
 
     async def filter_event(self, event):
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 7b8a3b9411..bbc9701d81 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -82,10 +82,10 @@ def set_expect_requests(self, expect_args={}, respond_args={}):
         def set_expect_requests_handler(self, expect_args=None, request_handler=None):
             self.httpserver.expect_request(expect_args).respond_with_handler(request_handler)
 
-        async def mock_dns(self, mock_data, scan=None):
+        async def mock_dns(self, mock_data, custom_lookup_fn=None, scan=None):
             if scan is None:
                 scan = self.scan
-            await scan.helpers.dns._mock_dns(mock_data)
+            await scan.helpers.dns._mock_dns(mock_data, custom_lookup_fn=custom_lookup_fn)
 
         def mock_interactsh(self, name):
             from ...conftest import Interactsh_mock
diff --git a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
index f6cb3b7400..c0bcb25a5b 100644
--- a/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
+++ b/bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py
@@ -84,3 +84,138 @@ def check(self, module_test, events):
             "asdf.www.blacklanternsecurity.com",
             "www.blacklanternsecurity.com",
         }
+
+
+class TestSubdomainEnumWildcardBaseline(ModuleTestBase):
+    # oh walmart.cn why are you like this
+    targets = ["www.walmart.cn"]
+    whitelist = ["walmart.cn"]
+    modules_overrides = []
+    config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}, "omit_event_types": []}
+    dedup_strategy = "highest_parent"
+
+    dns_mock_data = {
+        "walmart.cn": {"A": ["127.0.0.1"]},
+        "www.walmart.cn": {"A": ["127.0.0.1"]},
+        "test.walmart.cn": {"A": ["127.0.0.1"]},
+    }
+
+    async def setup_before_prep(self, module_test):
+        await module_test.mock_dns(self.dns_mock_data)
+        self.queries = []
+
+        async def mock_query(query):
+            self.queries.append(query)
+            return ["walmart.cn", "www.walmart.cn", "test.walmart.cn", "asdf.walmart.cn"]
+
+        # load subdomain enum template as module
+        from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+        subdomain_enum_module = subdomain_enum(module_test.scan)
+
+        subdomain_enum_module.query = mock_query
+        subdomain_enum_module._name = "subdomain_enum"
+        subdomain_enum_module.dedup_strategy = self.dedup_strategy
+        module_test.scan.modules["subdomain_enum"] = subdomain_enum_module
+
+    def check(self, module_test, events):
+        assert self.queries == ["walmart.cn"]
+        assert len(events) == 6
+        assert 2 == len(
+            [
+                e
+                for e in events
+                if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and str(e.module) == "A" and e.scope_distance == 1
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "www.walmart.cn"
+                and str(e.module) == "TARGET"
+                and e.scope_distance == 0
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "test.walmart.cn"
+                and str(e.module) == "subdomain_enum"
+                and e.scope_distance == 0
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME_UNRESOLVED"
+                and e.data == "asdf.walmart.cn"
+                and str(e.module) == "subdomain_enum"
+                and e.scope_distance == 0
+            ]
+        )
+
+
+class TestSubdomainEnumWildcardDefense(TestSubdomainEnumWildcardBaseline):
+    # oh walmart.cn why are you like this
+    targets = ["walmart.cn"]
+    modules_overrides = []
+    config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 10}}
+    dedup_strategy = "highest_parent"
+
+    dns_mock_data = {
+        "walmart.cn": {"A": ["127.0.0.2"], "TXT": ["asdf.walmart.cn"]},
+    }
+
+    async def setup_after_prep(self, module_test):
+        # simulate wildcard
+        custom_lookup = """
+def custom_lookup(query, rdtype):
+    import random
+    if rdtype == "A" and query.endswith(".walmart.cn"):
+        ip = ".".join([str(random.randint(0,256)) for _ in range(4)])
+        return {ip}
+"""
+        await module_test.mock_dns(self.dns_mock_data, custom_lookup_fn=custom_lookup)
+
+    def check(self, module_test, events):
+        # no subdomain enum should happen on this domain!
+        assert self.queries == []
+        assert len(events) == 6
+        assert 2 == len(
+            [e for e in events if e.type == "IP_ADDRESS" and str(e.module) == "A" and e.scope_distance == 1]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "walmart.cn"
+                and str(e.module) == "TARGET"
+                and e.scope_distance == 0
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME"
+                and e.data == "asdf.walmart.cn"
+                and str(e.module) == "TXT"
+                and e.scope_distance == 0
+                and "wildcard-possible" in e.tags
+                and "a-wildcard-possible" in e.tags
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "RAW_DNS_RECORD"
+                and e.data == {"host": "walmart.cn", "type": "TXT", "answer": '"asdf.walmart.cn"'}
+            ]
+        )

From dd70fa10355f46b3188a1dac88f97b4e7f46bd34 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Mon, 2 Sep 2024 00:10:26 -0400
Subject: [PATCH 11/12] make sure system nameservers are excluded from use by
 DNS brute force

---
 bbot/core/helpers/dns/brute.py             | 13 ++++++++++---
 bbot/defaults.yml                          |  3 +++
 bbot/test/bbot_fixtures.py                 | 11 ++++++++++-
 bbot/test/test_step_1/test_dns.py          | 11 ++++++++++-
 bbot/test/test_step_2/module_tests/base.py | 11 -----------
 5 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/bbot/core/helpers/dns/brute.py b/bbot/core/helpers/dns/brute.py
index 168815bf62..0c3799ca5b 100644
--- a/bbot/core/helpers/dns/brute.py
+++ b/bbot/core/helpers/dns/brute.py
@@ -15,15 +15,17 @@ class DNSBrute:
     >>> results = await self.helpers.dns.brute(self, domain, subdomains)
     """
 
-    nameservers_url = (
+    _nameservers_url = (
         "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
     )
 
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.log = logging.getLogger("bbot.helper.dns.brute")
+        self.dns_config = self.parent_helper.config.get("dns", {})
         self.num_canaries = 100
-        self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
+        self.max_resolvers = self.dns_config.get("brute_threads", 1000)
+        self.nameservers_url = self.dns_config.get("brute_nameservers", self._nameservers_url)
         self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
         self.digit_regex = self.parent_helper.re.compile(r"\d+")
         self._resolver_file = None
@@ -142,10 +144,15 @@ async def gen_subdomains(self, prefixes, domain):
 
     async def resolver_file(self):
         if self._resolver_file is None:
-            self._resolver_file = await self.parent_helper.wordlist(
+            self._resolver_file_original = await self.parent_helper.wordlist(
                 self.nameservers_url,
                 cache_hrs=24 * 7,
             )
+            nameservers = set(self.parent_helper.read_file(self._resolver_file_original))
+            nameservers.difference_update(self.parent_helper.dns.system_resolvers)
+            # exclude system nameservers from brute-force
+            # this helps prevent rate-limiting which might cause BBOT's main dns queries to fail
+            self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False)
         return self._resolver_file
 
     def gen_random_subdomains(self, n=50):
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index f577966723..18591bfdaa 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -34,6 +34,9 @@ dns:
   # How many concurrent DNS resolvers to use when brute-forcing
   # (under the hood this is passed through directly to massdns -s)
   brute_threads: 1000
+  # nameservers to use for DNS brute-forcing
+  # default is updated weekly and contains ~10K high-quality public servers
+  brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt
   # How far away from the main target to explore via DNS resolution (independent of scope.search_distance)
   # This is safe to change
   search_distance: 1
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
index 1c9631facd..abad144d12 100644
--- a/bbot/test/bbot_fixtures.py
+++ b/bbot/test/bbot_fixtures.py
@@ -15,7 +15,7 @@
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
-from bbot.core.helpers.misc import mkdir
+from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
 
 
@@ -33,6 +33,15 @@
 available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal"))
 
 
+def tempwordlist(content):
+    filename = bbot_test_dir / f"{rand_string(8)}"
+    with open(filename, "w", errors="ignore") as f:
+        for c in content:
+            line = f"{c}\n"
+            f.write(line)
+    return filename
+
+
 @pytest.fixture
 def clean_default_config(monkeypatch):
     clean_config = OmegaConf.merge(
diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py
index 648c5445b4..f4bfb218ae 100644
--- a/bbot/test/test_step_1/test_dns.py
+++ b/bbot/test/test_step_1/test_dns.py
@@ -742,7 +742,8 @@ async def test_dns_graph_structure(bbot_scanner):
     assert str(events_by_data["evilcorp.com"].module) == "host"
 
 
-def test_dns_helpers():
+@pytest.mark.asyncio
+async def test_dns_helpers(bbot_scanner):
     assert service_record("") == False
     assert service_record("localhost") == False
     assert service_record("www.example.com") == False
@@ -753,3 +754,11 @@ def test_dns_helpers():
     for srv_record in common_srvs[:100]:
         hostname = f"{srv_record}.example.com"
         assert service_record(hostname) == True
+
+    # make sure system nameservers are excluded from use by DNS brute force
+    brute_nameservers = tempwordlist(["1.2.3.4", "8.8.4.4", "4.3.2.1", "8.8.8.8"])
+    scan = bbot_scanner(config={"dns": {"brute_nameservers": brute_nameservers}})
+    scan.helpers.dns.system_resolvers = ["8.8.8.8", "8.8.4.4"]
+    resolver_file = await scan.helpers.dns.brute.resolver_file()
+    resolvers = set(scan.helpers.read_file(resolver_file))
+    assert resolvers == {"1.2.3.4", "4.3.2.1"}
diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index bbc9701d81..f5d4255f69 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -11,17 +11,6 @@
 log = logging.getLogger("bbot.test.modules")
 
 
-def tempwordlist(content):
-    from bbot.core.helpers.misc import rand_string
-
-    filename = bbot_test_dir / f"{rand_string(8)}"
-    with open(filename, "w", errors="ignore") as f:
-        for c in content:
-            line = f"{c}\n"
-            f.write(line)
-    return filename
-
-
 class ModuleTestBase:
     targets = ["blacklanternsecurity.com"]
     scan_name = None

From ecac174ec7633d3c7f2b063ed92dbb75e4cd5e59 Mon Sep 17 00:00:00 2001
From: github-actions <github-actions@github.com>
Date: Mon, 2 Sep 2024 20:16:06 -0400
Subject: [PATCH 12/12] cache yara regexes

---
 bbot/scanner/scanner.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index ba550f2177..111b6bf80b 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -231,6 +231,7 @@ def __init__(
 
         self._dns_strings = None
         self._dns_regexes = None
+        self._dns_regexes_yara = None
 
         self.__log_handlers = None
         self._log_handler_backup = []
@@ -1000,7 +1001,7 @@ def dns_regexes(self):
             ...     for match in regex.finditer(response.text):
             ...         hostname = match.group().lower()
         """
-        if not self._dns_regexes:
+        if self._dns_regexes is None:
             self._dns_regexes = self._generate_dns_regexes(r"((?:(?:[\w-]+)\.)+")
         return self._dns_regexes
 
@@ -1009,7 +1010,9 @@ def dns_regexes_yara(self):
         """
         Returns a list of DNS hostname regexes formatted specifically for compatibility with YARA rules.
         """
-        return self._generate_dns_regexes(r"(([a-z0-9-]+\.)+")
+        if self._dns_regexes_yara is None:
+            self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)+")
+        return self._dns_regexes_yara
 
     @property
     def json(self):