Merge pull request #1723 from blacklanternsecurity/dns-optimizations-2

DNS Optimizations

bbot/core/event/base.py

@@ -171,6 +171,7 @@ def __init__(
         self._module_priority = None
         self._resolved_hosts = set()
         self.dns_children = dict()
+        self.raw_dns_records = dict()
         self._discovery_context = ""
         self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
         self.web_spider_distance = 0
@@ -1068,6 +1069,17 @@ def __init__(self, *args, **kwargs):
             if parent_module_type == "DNS":
                 self.dns_resolve_distance += 1
         # self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
+        # tag subdomain / domain
+        if is_subdomain(self.host):
+            self.add_tag("subdomain")
+        elif is_domain(self.host):
+            self.add_tag("domain")
+        # tag private IP
+        try:
+            if self.host.is_private:
+                self.add_tag("private-ip")
+        except AttributeError:
+            pass
 
 
 class IP_RANGE(DnsEvent):
@@ -1084,13 +1096,6 @@ def _host(self):
 
 
 class DNS_NAME(DnsEvent):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if is_subdomain(self.data):
-            self.add_tag("subdomain")
-        elif is_domain(self.data):
-            self.add_tag("domain")
-
     def sanitize_data(self, data):
         return validators.validate_host(data)
 
@@ -1513,7 +1518,7 @@ class FILESYSTEM(DictPathEvent):
     pass
 
 
-class RAW_DNS_RECORD(DictHostEvent):
+class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
     # don't emit raw DNS records for affiliates
     _always_emit_tags = ["target"]
 

bbot/core/helpers/dns/brute.py

@@ -15,15 +15,17 @@ class DNSBrute:
     >>> results = await self.helpers.dns.brute(self, domain, subdomains)
     """
 
-    nameservers_url = (
+    _nameservers_url = (
         "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
     )
 
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.log = logging.getLogger("bbot.helper.dns.brute")
+        self.dns_config = self.parent_helper.config.get("dns", {})
         self.num_canaries = 100
-        self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
+        self.max_resolvers = self.dns_config.get("brute_threads", 1000)
+        self.nameservers_url = self.dns_config.get("brute_nameservers", self._nameservers_url)
         self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
         self.digit_regex = self.parent_helper.re.compile(r"\d+")
         self._resolver_file = None
@@ -39,18 +41,12 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
             type = "A"
         type = str(type).strip().upper()
 
-        domain_wildcard_rdtypes = set()
-        for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items():
-            for rdtype, results in rdtypes.items():
-                if results:
-                    domain_wildcard_rdtypes.add(rdtype)
-        if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]):
-            self.log.info(
-                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
+        wildcard_rdtypes = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME"))
+        if wildcard_rdtypes:
+            self.log.hugewarning(
+                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})"
             )
             return []
-        else:
-            self.log.debug(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
 
         canaries = self.gen_random_subdomains(self.num_canaries)
         canaries_list = list(canaries)
@@ -148,10 +144,15 @@ async def gen_subdomains(self, prefixes, domain):
 
     async def resolver_file(self):
         if self._resolver_file is None:
-            self._resolver_file = await self.parent_helper.wordlist(
+            self._resolver_file_original = await self.parent_helper.wordlist(
                 self.nameservers_url,
                 cache_hrs=24 * 7,
             )
+            nameservers = set(self.parent_helper.read_file(self._resolver_file_original))
+            nameservers.difference_update(self.parent_helper.dns.system_resolvers)
+            # exclude system nameservers from brute-force
+            # this helps prevent rate-limiting which might cause BBOT's main dns queries to fail
+            self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False)
         return self._resolver_file
 
     def gen_random_subdomains(self, n=50):

bbot/core/helpers/dns/dns.py

@@ -66,7 +66,7 @@ def __init__(self, parent_helper):
         self.resolver.timeout = self.timeout
         self.resolver.lifetime = self.timeout
 
-        self.runaway_limit = self.config.get("runaway_limit", 5)
+        self.runaway_limit = self.dns_config.get("runaway_limit", 5)
 
         # wildcard handling
         self.wildcard_disable = self.dns_config.get("wildcard_disable", False)
@@ -117,8 +117,11 @@ def brute(self):
             self._brute = DNSBrute(self.parent_helper)
         return self._brute
 
-    @async_cachedmethod(lambda self: self._is_wildcard_cache)
-    async def is_wildcard(self, query, ips=None, rdtype=None):
+    @async_cachedmethod(
+        lambda self: self._is_wildcard_cache,
+        key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)),
+    )
+    async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         """
         Use this method to check whether a *host* is a wildcard entry
 
@@ -150,9 +153,6 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
         Note:
             `is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)
         """
-        if [ips, rdtype].count(None) == 1:
-            raise ValueError("Both ips and rdtype must be specified")
-
         query = self._wildcard_prevalidation(query)
         if not query:
             return {}
@@ -161,15 +161,17 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
         if is_domain(query):
             return {}
 
-        return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype)
+        return await self.run_and_return("is_wildcard", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records)
 
-    @async_cachedmethod(lambda self: self._is_wildcard_domain_cache)
-    async def is_wildcard_domain(self, domain, log_info=False):
+    @async_cachedmethod(
+        lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes)))
+    )
+    async def is_wildcard_domain(self, domain, rdtypes):
         domain = self._wildcard_prevalidation(domain)
         if not domain:
             return {}
 
-        return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False)
+        return await self.run_and_return("is_wildcard_domain", domain=domain, rdtypes=rdtypes)
 
     def _wildcard_prevalidation(self, host):
         if self.wildcard_disable:
@@ -192,8 +194,8 @@ def _wildcard_prevalidation(self, host):
 
         return host
 
-    async def _mock_dns(self, mock_data):
+    async def _mock_dns(self, mock_data, custom_lookup_fn=None):
         from .mock import MockResolver
 
-        self.resolver = MockResolver(mock_data)
-        await self.run_and_return("_mock_dns", mock_data=mock_data)
+        self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn)
+        await self.run_and_return("_mock_dns", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn)