make sure system nameservers are excluded from use by DNS brute force

bbot/core/helpers/dns/brute.py

@@ -15,15 +15,17 @@ class DNSBrute:
     >>> results = await self.helpers.dns.brute(self, domain, subdomains)
     """
 
-    nameservers_url = (
+    _nameservers_url = (
         "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
     )
 
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.log = logging.getLogger("bbot.helper.dns.brute")
+        self.dns_config = self.parent_helper.config.get("dns", {})
         self.num_canaries = 100
-        self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
+        self.max_resolvers = self.dns_config.get("brute_threads", 1000)
+        self.nameservers_url = self.dns_config.get("brute_nameservers", self._nameservers_url)
         self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
         self.digit_regex = self.parent_helper.re.compile(r"\d+")
         self._resolver_file = None
@@ -142,10 +144,15 @@ async def gen_subdomains(self, prefixes, domain):
 
     async def resolver_file(self):
         if self._resolver_file is None:
-            self._resolver_file = await self.parent_helper.wordlist(
+            self._resolver_file_original = await self.parent_helper.wordlist(
                 self.nameservers_url,
                 cache_hrs=24 * 7,
             )
+            nameservers = set(self.parent_helper.read_file(self._resolver_file_original))
+            nameservers.difference_update(self.parent_helper.dns.system_resolvers)
+            # exclude system nameservers from brute-force
+            # this helps prevent rate-limiting which might cause BBOT's main dns queries to fail
+            self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False)
         return self._resolver_file
 
     def gen_random_subdomains(self, n=50):

bbot/defaults.yml

@@ -34,6 +34,9 @@ dns:
   # How many concurrent DNS resolvers to use when brute-forcing
   # (under the hood this is passed through directly to massdns -s)
   brute_threads: 1000
+  # nameservers to use for DNS brute-forcing
+  # default is updated weekly and contains ~10K high-quality public servers
+  brute_nameservers: https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt
   # How far away from the main target to explore via DNS resolution (independent of scope.search_distance)
   # This is safe to change
   search_distance: 1

bbot/test/bbot_fixtures.py

@@ -15,7 +15,7 @@
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
-from bbot.core.helpers.misc import mkdir
+from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
 
 
@@ -33,6 +33,15 @@
 available_internal_modules = list(DEFAULT_PRESET.module_loader.configs(type="internal"))
 
 
+def tempwordlist(content):
+    filename = bbot_test_dir / f"{rand_string(8)}"
+    with open(filename, "w", errors="ignore") as f:
+        for c in content:
+            line = f"{c}\n"
+            f.write(line)
+    return filename
+
+
 @pytest.fixture
 def clean_default_config(monkeypatch):
     clean_config = OmegaConf.merge(

bbot/test/test_step_1/test_dns.py

@@ -742,7 +742,8 @@ async def test_dns_graph_structure(bbot_scanner):
     assert str(events_by_data["evilcorp.com"].module) == "host"
 
 
-def test_dns_helpers():
+@pytest.mark.asyncio
+async def test_dns_helpers(bbot_scanner):
     assert service_record("") == False
     assert service_record("localhost") == False
     assert service_record("www.example.com") == False
@@ -753,3 +754,11 @@ def test_dns_helpers():
     for srv_record in common_srvs[:100]:
         hostname = f"{srv_record}.example.com"
         assert service_record(hostname) == True
+
+    # make sure system nameservers are excluded from use by DNS brute force
+    brute_nameservers = tempwordlist(["1.2.3.4", "8.8.4.4", "4.3.2.1", "8.8.8.8"])
+    scan = bbot_scanner(config={"dns": {"brute_nameservers": brute_nameservers}})
+    scan.helpers.dns.system_resolvers = ["8.8.8.8", "8.8.4.4"]
+    resolver_file = await scan.helpers.dns.brute.resolver_file()
+    resolvers = set(scan.helpers.read_file(resolver_file))
+    assert resolvers == {"1.2.3.4", "4.3.2.1"}

bbot/test/test_step_2/module_tests/base.py

@@ -11,17 +11,6 @@
 log = logging.getLogger("bbot.test.modules")
 
 
-def tempwordlist(content):
-    from bbot.core.helpers.misc import rand_string
-
-    filename = bbot_test_dir / f"{rand_string(8)}"
-    with open(filename, "w", errors="ignore") as f:
-        for c in content:
-            line = f"{c}\n"
-            f.write(line)
-    return filename
-
-
 class ModuleTestBase:
     targets = ["blacklanternsecurity.com"]
     scan_name = None