Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DNS Optimizations #1723

Merged
merged 12 commits into from
Sep 3, 2024
21 changes: 13 additions & 8 deletions bbot/core/event/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@ def __init__(
self._module_priority = None
self._resolved_hosts = set()
self.dns_children = dict()
self.raw_dns_records = dict()
self._discovery_context = ""
self._discovery_context_regex = re.compile(r"\{(?:event|module)[^}]*\}")
self.web_spider_distance = 0
Expand Down Expand Up @@ -1053,6 +1054,17 @@ def __init__(self, *args, **kwargs):
if parent_module_type == "DNS":
self.dns_resolve_distance += 1
# self.add_tag(f"resolve-distance-{self.dns_resolve_distance}")
# tag subdomain / domain
if is_subdomain(self.host):
self.add_tag("subdomain")
elif is_domain(self.host):
self.add_tag("domain")
# tag private IP
try:
if self.host.is_private:
self.add_tag("private-ip")
except AttributeError:
pass


class IP_RANGE(DnsEvent):
Expand All @@ -1069,13 +1081,6 @@ def _host(self):


class DNS_NAME(DnsEvent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if is_subdomain(self.data):
self.add_tag("subdomain")
elif is_domain(self.data):
self.add_tag("domain")

def sanitize_data(self, data):
return validators.validate_host(data)

Expand Down Expand Up @@ -1498,7 +1503,7 @@ class FILESYSTEM(DictPathEvent):
pass


class RAW_DNS_RECORD(DictHostEvent):
class RAW_DNS_RECORD(DictHostEvent, DnsEvent):
# don't emit raw DNS records for affiliates
_always_emit_tags = ["target"]

Expand Down
27 changes: 14 additions & 13 deletions bbot/core/helpers/dns/brute.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,17 @@ class DNSBrute:
>>> results = await self.helpers.dns.brute(self, domain, subdomains)
"""

nameservers_url = (
_nameservers_url = (
"https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt"
)

def __init__(self, parent_helper):
self.parent_helper = parent_helper
self.log = logging.getLogger("bbot.helper.dns.brute")
self.dns_config = self.parent_helper.config.get("dns", {})
self.num_canaries = 100
self.max_resolvers = self.parent_helper.config.get("dns", {}).get("brute_threads", 1000)
self.max_resolvers = self.dns_config.get("brute_threads", 1000)
self.nameservers_url = self.dns_config.get("brute_nameservers", self._nameservers_url)
self.devops_mutations = list(self.parent_helper.word_cloud.devops_mutations)
self.digit_regex = self.parent_helper.re.compile(r"\d+")
self._resolver_file = None
Expand All @@ -39,18 +41,12 @@ async def dnsbrute(self, module, domain, subdomains, type=None):
type = "A"
type = str(type).strip().upper()

domain_wildcard_rdtypes = set()
for _domain, rdtypes in (await self.parent_helper.dns.is_wildcard_domain(domain)).items():
for rdtype, results in rdtypes.items():
if results:
domain_wildcard_rdtypes.add(rdtype)
if any([r in domain_wildcard_rdtypes for r in (type, "CNAME")]):
self.log.info(
f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(domain_wildcard_rdtypes)})"
wildcard_rdtypes = await self.parent_helper.dns.is_wildcard_domain(domain, (type, "CNAME"))
if wildcard_rdtypes:
self.log.hugewarning(
f"Aborting massdns on {domain} because it's a wildcard domain ({','.join(wildcard_rdtypes)})"
)
return []
else:
self.log.debug(f"{domain}: A is not in domain_wildcard_rdtypes:{domain_wildcard_rdtypes}")

canaries = self.gen_random_subdomains(self.num_canaries)
canaries_list = list(canaries)
Expand Down Expand Up @@ -148,10 +144,15 @@ async def gen_subdomains(self, prefixes, domain):

async def resolver_file(self):
if self._resolver_file is None:
self._resolver_file = await self.parent_helper.wordlist(
self._resolver_file_original = await self.parent_helper.wordlist(
self.nameservers_url,
cache_hrs=24 * 7,
)
nameservers = set(self.parent_helper.read_file(self._resolver_file_original))
nameservers.difference_update(self.parent_helper.dns.system_resolvers)
# exclude system nameservers from brute-force
# this helps prevent rate-limiting which might cause BBOT's main dns queries to fail
self._resolver_file = self.parent_helper.tempfile(nameservers, pipe=False)
return self._resolver_file

def gen_random_subdomains(self, n=50):
Expand Down
28 changes: 15 additions & 13 deletions bbot/core/helpers/dns/dns.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def __init__(self, parent_helper):
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout

self.runaway_limit = self.config.get("runaway_limit", 5)
self.runaway_limit = self.dns_config.get("runaway_limit", 5)

# wildcard handling
self.wildcard_disable = self.dns_config.get("wildcard_disable", False)
Expand Down Expand Up @@ -117,8 +117,11 @@ def brute(self):
self._brute = DNSBrute(self.parent_helper)
return self._brute

@async_cachedmethod(lambda self: self._is_wildcard_cache)
async def is_wildcard(self, query, ips=None, rdtype=None):
@async_cachedmethod(
lambda self: self._is_wildcard_cache,
key=lambda query, rdtypes, raw_dns_records: (query, tuple(sorted(rdtypes)), bool(raw_dns_records)),
)
async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
"""
Use this method to check whether a *host* is a wildcard entry

Expand Down Expand Up @@ -150,9 +153,6 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
Note:
`is_wildcard` can be True, False, or None (indicating that wildcard detection was inconclusive)
"""
if [ips, rdtype].count(None) == 1:
raise ValueError("Both ips and rdtype must be specified")

query = self._wildcard_prevalidation(query)
if not query:
return {}
Expand All @@ -161,15 +161,17 @@ async def is_wildcard(self, query, ips=None, rdtype=None):
if is_domain(query):
return {}

return await self.run_and_return("is_wildcard", query=query, ips=ips, rdtype=rdtype)
return await self.run_and_return("is_wildcard", query=query, rdtypes=rdtypes, raw_dns_records=raw_dns_records)

@async_cachedmethod(lambda self: self._is_wildcard_domain_cache)
async def is_wildcard_domain(self, domain, log_info=False):
@async_cachedmethod(
lambda self: self._is_wildcard_domain_cache, key=lambda domain, rdtypes: (domain, tuple(sorted(rdtypes)))
)
async def is_wildcard_domain(self, domain, rdtypes):
domain = self._wildcard_prevalidation(domain)
if not domain:
return {}

return await self.run_and_return("is_wildcard_domain", domain=domain, log_info=False)
return await self.run_and_return("is_wildcard_domain", domain=domain, rdtypes=rdtypes)

def _wildcard_prevalidation(self, host):
if self.wildcard_disable:
Expand All @@ -192,8 +194,8 @@ def _wildcard_prevalidation(self, host):

return host

async def _mock_dns(self, mock_data):
async def _mock_dns(self, mock_data, custom_lookup_fn=None):
from .mock import MockResolver

self.resolver = MockResolver(mock_data)
await self.run_and_return("_mock_dns", mock_data=mock_data)
self.resolver = MockResolver(mock_data, custom_lookup_fn=custom_lookup_fn)
await self.run_and_return("_mock_dns", mock_data=mock_data, custom_lookup_fn=custom_lookup_fn)
Loading
Loading