Helperify Massdns

bbot/core/helpers/bloom.py

@@ -0,0 +1,71 @@
+import os
+import mmh3
+import mmap
+
+
+class BloomFilter:
+    """
+    Simple bloom filter implementation capable of rougly 400K lookups/s.
+
+    BBOT uses bloom filters in scenarios like DNS brute-forcing, where it's useful to keep track
+    of which mutations have been tried so far.
+
+    A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.
+    A python hash is 36 bytes. So if you wanted to store these in a set, this would take up
+    36 * 10M * 2 (key+value) == 720 megabytes. So we save rougly 7 times the space.
+    """
+
+    def __init__(self, size=8000000):
+        self.size = size  # total bits
+        self.byte_size = (size + 7) // 8  # calculate byte size needed for the given number of bits
+
+        # Create an anonymous mmap region, compatible with both Windows and Unix
+        if os.name == "nt":  # Windows
+            # -1 indicates an anonymous memory map in Windows
+            self.mmap_file = mmap.mmap(-1, self.byte_size)
+        else:  # Unix/Linux
+            # Use MAP_ANONYMOUS along with MAP_SHARED
+            self.mmap_file = mmap.mmap(-1, self.byte_size, prot=mmap.PROT_WRITE, flags=mmap.MAP_ANON | mmap.MAP_SHARED)
+
+        self.clear_all_bits()
+
+    def add(self, item):
+        for hash_value in self._hashes(item):
+            index = hash_value // 8
+            position = hash_value % 8
+            current_byte = self.mmap_file[index]
+            self.mmap_file[index] = current_byte | (1 << position)
+
+    def check(self, item):
+        for hash_value in self._hashes(item):
+            index = hash_value // 8
+            position = hash_value % 8
+            current_byte = self.mmap_file[index]
+            if not (current_byte & (1 << position)):
+                return False
+        return True
+
+    def clear_all_bits(self):
+        self.mmap_file.seek(0)
+        # Write zeros across the entire mmap length
+        self.mmap_file.write(b"\x00" * self.byte_size)
+
+    def _hashes(self, item):
+        if not isinstance(item, bytes):
+            if not isinstance(item, str):
+                item = str(item)
+            item = item.encode("utf-8")
+        return [abs(hash(item)) % self.size, abs(mmh3.hash(item)) % self.size, abs(self._fnv1a_hash(item)) % self.size]
+
+    def _fnv1a_hash(self, data):
+        hash = 0x811C9DC5  # 2166136261
+        for byte in data:
+            hash ^= byte
+            hash = (hash * 0x01000193) % 2**32  # 16777619
+        return hash
+
+    def __del__(self):
+        self.mmap_file.close()
+
+    def __contains__(self, item):
+        return self.check(item)

bbot/core/helpers/dns/brute.py

@@ -0,0 +1,180 @@
+import json
+import random
+import asyncio
+import logging
+import subprocess
+
+
+class DNSBrute:
+    """
+    Helper for DNS brute-forcing.
+
+    Examples:
+    >>> domain = "evilcorp.com"
+    >>> subdomains = ["www", "mail"]
+    >>> results = await self.helpers.dns.brute(self, domain, subdomains)
+    """
+
+    nameservers_url = (
+        "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
+    )
+
+    def __init__(self, parent_helper):
+        self.parent_helper = parent_helper
+        self.log = logging.getLogger("bbot.helper.dns.brute")
+        self.num_canaries = 100
+        self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
+        self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
+        self.digit_regex = self.parent_helper.re.compile(r"\d+")
+        self._resolver_file = None
+        self._dnsbrute_lock = asyncio.Lock()
+
+    async def __call__(self, *args, **kwargs):
+        return await self.dnsbrute(*args, **kwargs)
+
+    async def dnsbrute(self, module, domain, subdomains, type=None):
+        subdomains = list(subdomains)
+
+        if type is None:
+            type = "A"
+        type = str(type).strip().upper()
+
+        domain_wildcard_rdtypes = set()
+        for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items():
+            for rdtype, results in rdtypes.items():
+                if results:
+                    domain_wildcard_rdtypes.add(rdtype)
+        if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]):
+            self.log.info(
+                f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
+            )
+            return []
+        else:
+            self.log.trace(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")
+
+        canaries = self.gen_random_subdomains(self.num_canaries)
+        canaries_list = list(canaries)
+        canaries_pre = canaries_list[: int(self.num_canaries / 2)]
+        canaries_post = canaries_list[int(self.num_canaries / 2) :]
+        # sandwich subdomains between canaries
+        subdomains = canaries_pre + subdomains + canaries_post
+
+        results = []
+        canaries_triggered = []
+        async for hostname, ip, rdtype in self._massdns(module, domain, subdomains, rdtype=type):
+            sub = hostname.split(domain)[0]
+            if sub in canaries:
+                canaries_triggered.append(sub)
+            else:
+                results.append(hostname)
+
+        if len(canaries_triggered) > 5:
+            self.log.info(
+                f"Aborting massdns on {domain} due to false positive: ({len(canaries_triggered):,} canaries triggered - {','.join(canaries_triggered)})"
+            )
+            return []
+
+        # everything checks out
+        return results
+
+    async def _massdns(self, module, domain, subdomains, rdtype):
+        """
+        {
+            "name": "www.blacklanternsecurity.com.",
+            "type": "A",
+            "class": "IN",
+            "status": "NOERROR",
+            "data": {
+            "answers": [
+                {
+                "ttl": 3600,
+                "type": "CNAME",
+                "class": "IN",
+                "name": "www.blacklanternsecurity.com.",
+                "data": "blacklanternsecurity.github.io."
+                },
+                {
+                "ttl": 3600,
+                "type": "A",
+                "class": "IN",
+                "name": "blacklanternsecurity.github.io.",
+                "data": "185.199.108.153"
+                }
+            ]
+            },
+            "resolver": "168.215.165.186:53"
+        }
+        """
+        resolver_file = await self.resolver_file()
+        command = (
+            "massdns",
+            "-r",
+            resolver_file,
+            "-s",
+            self.max_resolvers,
+            "-t",
+            rdtype,
+            "-o",
+            "J",
+            "-q",
+        )
+        subdomains = self.gen_subdomains(subdomains, domain)
+        hosts_yielded = set()
+        async with self._dnsbrute_lock:
+            async for line in module.run_process_live(*command, stderr=subprocess.DEVNULL, input=subdomains):
+                try:
+                    j = json.loads(line)
+                except json.decoder.JSONDecodeError:
+                    self.log.debug(f"Failed to decode line: {line}")
+                    continue
+                answers = j.get("data", {}).get("answers", [])
+                if type(answers) == list and len(answers) > 0:
+                    answer = answers[0]
+                    hostname = answer.get("name", "").strip(".").lower()
+                    if hostname.endswith(f".{domain}"):
+                        data = answer.get("data", "")
+                        rdtype = answer.get("type", "").upper()
+                        if data and rdtype:
+                            hostname_hash = hash(hostname)
+                            if hostname_hash not in hosts_yielded:
+                                hosts_yielded.add(hostname_hash)
+                                yield hostname, data, rdtype
+
+    async def gen_subdomains(self, prefixes, domain):
+        for p in prefixes:
+            if domain:
+                p = f"{p}.{domain}"
+            yield p
+
+    async def resolver_file(self):
+        if self._resolver_file is None:
+            self._resolver_file = await self.parent_helper.wordlist(
+                self.nameservers_url,
+                cache_hrs=24 * 7,
+            )
+        return self._resolver_file
+
+    def gen_random_subdomains(self, n=50):
+        delimiters = (".", "-")
+        lengths = list(range(3, 8))
+        for i in range(0, max(0, n - 5)):
+            d = delimiters[i % len(delimiters)]
+            l = lengths[i % len(lengths)]
+            segments = list(random.choice(self.devops_mutations) for _ in range(l))
+            segments.append(self.parent_helper.rand_string(length=8, digits=False))
+            subdomain = d.join(segments)
+            yield subdomain
+        for _ in range(5):
+            yield self.parent_helper.rand_string(length=8, digits=False)
+
+    def has_excessive_digits(self, d):
+        """
+        Identifies dns names with excessive numbers, e.g.:
+            - w1-2-3.evilcorp.com
+            - ptr1234.evilcorp.com
+        """
+        is_ptr = self.parent_helper.is_ptr(d)
+        digits = self.digit_regex.findall(d)
+        excessive_digits = len(digits) > 2
+        long_digits = any(len(d) > 3 for d in digits)
+        return is_ptr or excessive_digits or long_digits

bbot/core/helpers/dns/dns.py

@@ -73,6 +73,9 @@ def __init__(self, parent_helper):
         # TODO: DNS server speed test (start in background task)
         self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False)
 
+        # brute force helper
+        self._brute = None
+
     async def resolve(self, query, **kwargs):
         return await self.run_and_return("resolve", query=query, **kwargs)
 
@@ -84,6 +87,14 @@ async def resolve_raw_batch(self, queries):
         async for _ in self.run_and_yield("resolve_raw_batch", queries=queries):
             yield _
 
+    @property
+    def brute(self):
+        if self._brute is None:
+            from .brute import DNSBrute
+
+            self._brute = DNSBrute(self.parent_helper)
+        return self._brute
+
     async def is_wildcard(self, query, ips=None, rdtype=None):
         """
         Use this method to check whether a *host* is a wildcard entry

bbot/core/helpers/helper.py

@@ -91,6 +91,11 @@ def __init__(self, preset):
         self.word_cloud = WordCloud(self)
         self.dummy_modules = {}
 
+    def bloom_filter(self, size):
+        from .bloom import BloomFilter
+
+        return BloomFilter(size)
+
     def interactsh(self, *args, **kwargs):
         return Interactsh(self, *args, **kwargs)
 
@@ -110,8 +115,8 @@ def clean_old_scans(self):
         _filter = lambda x: x.is_dir() and self.regexes.scan_name_regex.match(x.name)
         self.clean_old(self.scans_dir, keep=self.keep_old_scans, filter=_filter)
 
-    def make_target(self, *events):
-        return Target(*events)
+    def make_target(self, *events, **kwargs):
+        return Target(*events, **kwargs)
 
     @property
     def config(self):

bbot/core/helpers/wordcloud.py

@@ -451,7 +451,7 @@ def add_word(self, word):
 
 class DNSMutator(Mutator):
     """
-    DNS-specific mutator used by the `massdns` module to generate target-specific subdomain mutations.
+    DNS-specific mutator used by the `dnsbrute_mutations` module to generate target-specific subdomain mutations.
 
     This class extends the Mutator base class to add DNS-specific logic for generating
     subdomain mutations based on input words. It utilizes custom word extraction patterns

bbot/defaults.yml

@@ -24,6 +24,10 @@ deps:
   ffuf:
     version: "2.1.0"
 
+dns:
+  # Number of concurrent massdns lookups (-s)
+  brute_threads: 1000
+
 ### WEB SPIDER ###
 
 # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed)

bbot/modules/base.py

@@ -1461,6 +1461,10 @@ async def _worker(self):
             except asyncio.CancelledError:
                 self.log.trace("Worker cancelled")
                 raise
+            except BaseException as e:
+                self.critical(f"Critical failure in intercept module {self.name}: {e}")
+                self.critical(traceback.format_exc())
+                self.scan.stop()
         self.log.trace(f"Worker stopped")
 
     async def get_incoming_event(self):

bbot/modules/dnsbrute.py

@@ -0,0 +1,49 @@
+from bbot.modules.templates.subdomain_enum import subdomain_enum
+
+
+class dnsbrute(subdomain_enum):
+    flags = ["subdomain-enum", "passive", "aggressive"]
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    meta = {"description": "Brute-force subdomains with massdns + static wordlist"}
+    options = {
+        "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt",
+        "max_depth": 5,
+    }
+    options_desc = {
+        "wordlist": "Subdomain wordlist URL",
+        "max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com",
+    }
+    deps_common = ["massdns"]
+    reject_wildcards = "strict"
+    dedup_strategy = "lowest_parent"
+    _qsize = 10000
+
+    async def setup(self):
+        self.max_depth = max(1, self.config.get("max_depth", 5))
+        self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist"))
+        self.subdomain_list = set(self.helpers.read_file(self.subdomain_file))
+        return await super().setup()
+
+    async def filter_event(self, event):
+        eligible, reason = await super().filter_event(event)
+        query = self.make_query(event)
+
+        # limit brute force depth
+        subdomain_depth = self.helpers.subdomain_depth(query) + 1
+        if subdomain_depth > self.max_depth:
+            eligible = False
+            reason = f"subdomain depth of *.{query} ({subdomain_depth}) > max_depth ({self.max_depth})"
+
+        # don't brute-force things that look like autogenerated PTRs
+        if self.helpers.dns.brute.has_excessive_digits(query):
+            eligible = False
+            reason = f'"{query}" looks like an autogenerated PTR'
+
+        return eligible, reason
+
+    async def handle_event(self, event):
+        query = self.make_query(event)
+        self.info(f"Brute-forcing subdomains for {query} (source: {event.data})")
+        for hostname in await self.helpers.dns.brute(self, query, self.subdomain_list):
+            await self.emit_event(hostname, "DNS_NAME", source=event)