From ef3314c96e1eff30a758b9e449335c31f5e3d9f4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 7 Aug 2023 12:23:14 -0400 Subject: [PATCH 01/29] better dns/http rate limiting --- bbot/core/helpers/cache.py | 3 + bbot/core/helpers/dns.py | 109 ++++++++++++++++----------- bbot/core/helpers/names_generator.py | 1 + bbot/core/helpers/web.py | 11 ++- bbot/modules/massdns.py | 2 +- bbot/test/test_step_1/test_dns.py | 11 ++- poetry.lock | 18 +++-- pyproject.toml | 2 +- 8 files changed, 100 insertions(+), 57 deletions(-) diff --git a/bbot/core/helpers/cache.py b/bbot/core/helpers/cache.py index 15c9d904f..3eb54daf7 100644 --- a/bbot/core/helpers/cache.py +++ b/bbot/core/helpers/cache.py @@ -109,6 +109,9 @@ def values(self): def items(self): return self._cache.items() + def clear(self): + return self._cache.clear() + def _hash(self, v): if type(v) == int: return v diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 6046c351c..914144e91 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -1,3 +1,4 @@ +import time import asyncio import logging import ipaddress @@ -16,9 +17,21 @@ log = logging.getLogger("bbot.core.helpers.dns") +class BBOTAsyncResolver(dns.asyncresolver.Resolver): + def __init__(self, *args, **kwargs): + self._parent_helper = kwargs.pop("_parent_helper") + dns_queries_per_second = self._parent_helper.config.get("dns_queries_per_second", 100) + self._dns_rate_limiter = RateLimiter(dns_queries_per_second, "DNS") + super().__init__(*args, **kwargs) + + async def resolve(self, *args, **kwargs): + async with self._dns_rate_limiter: + return await super().resolve(*args, **kwargs) + + class DNSHelper: """ - For automatic wildcard detection, nameserver validation, etc. + For host resolution, automatic wildcard detection, etc. """ all_rdtypes = ["A", "AAAA", "SRV", "MX", "NS", "SOA", "CNAME", "TXT"] @@ -26,7 +39,7 @@ class DNSHelper: def __init__(self, parent_helper): self.parent_helper = parent_helper try: - self.resolver = dns.asyncresolver.Resolver() + self.resolver = BBOTAsyncResolver(_parent_helper=self.parent_helper) except Exception as e: raise DNSError(f"Failed to create BBOT DNS resolver: {e}") self.timeout = self.parent_helper.config.get("dns_timeout", 5) @@ -62,12 +75,13 @@ def __init__(self, parent_helper): # since wildcard detection takes some time, This is to prevent multiple # modules from kicking off wildcard detection for the same domain at the same time self._wildcard_lock = NamedLock() + self._dns_connectivity_lock = asyncio.Lock() + self._last_dns_success = None + self._last_connectivity_warning = time.time() # keeps track of warnings issued for wildcard detection to prevent duplicate warnings self._dns_warnings = set() self._errors = dict() self.fallback_nameservers_file = self.parent_helper.wordlist_dir / "nameservers.txt" - self.dns_queries_per_second = self.parent_helper.config.get("dns_queries_per_second", 100) - self.dns_rate_limiter = RateLimiter(self.dns_queries_per_second, "DNS") self._debug = self.parent_helper.config.get("dns_debug", False) self._dummy_modules = dict() self._dns_cache = self.parent_helper.CacheDict(max_size=100000) @@ -158,45 +172,30 @@ async def _resolve_hostname(self, query, **kwargs): parent = self.parent_helper.parent_domain(query) retries = kwargs.pop("retries", self.retries) - cache_result = kwargs.pop("cache_result", False) + use_cache = kwargs.pop("use_cache", True) tries_left = int(retries) + 1 parent_hash = hash(f"{parent}:{rdtype}") dns_cache_hash = hash(f"{query}:{rdtype}") while tries_left > 0: try: - try: - results = self._dns_cache[dns_cache_hash] - except KeyError: + if use_cache: + results = self._dns_cache.get(dns_cache_hash, []) + if not results: error_count = self._errors.get(parent_hash, 0) if error_count >= self.abort_threshold: - try: - dns_server_working = list( - await asyncio.wait_for( - self.resolver.resolve("www.google.com", rdtype="A"), self.timeout + 10 - ) - ) - except Exception: - dns_server_working = [] - if not dns_server_working: - log.warning(f"DNS queries are failing, you may be blasting a little too hard") - self._errors.clear() - else: - log.info( + connectivity = await self._connectivity_check() + if connectivity: + log.verbose( f'Aborting query "{query}" because failed {rdtype} queries for "{parent}" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' ) if parent_hash not in self._dns_warnings: - log.warning( + log.info( f'Aborting future {rdtype} queries to "{parent}" because error count ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' ) self._dns_warnings.add(parent_hash) return results, errors - async with self.dns_rate_limiter: - # wait_for exists here because of this: - # https://github.com/rthalley/dnspython/issues/976 - results = await asyncio.wait_for( - self._catch(self.resolver.resolve, query, **kwargs), self.timeout + 10 - ) - if cache_result: + results = await self._catch(self.resolver.resolve, query, **kwargs) + if use_cache: self._dns_cache[dns_cache_hash] = results if parent_hash in self._errors: self._errors[parent_hash] = 0 @@ -226,26 +225,26 @@ async def _resolve_hostname(self, query, **kwargs): else: log.verbose(err_msg) + if results: + self._last_dns_success = time.time() + return results, errors async def _resolve_ip(self, query, **kwargs): self.debug(f"Reverse-resolving {query} with kwargs={kwargs}") retries = kwargs.pop("retries", 0) - cache_result = kwargs.pop("cache_result", False) + use_cache = kwargs.pop("use_cache", True) tries_left = int(retries) + 1 results = [] errors = [] dns_cache_hash = hash(f"{query}:PTR") while tries_left > 0: try: - try: - results = self._dns_cache[dns_cache_hash] - except KeyError: - async with self.dns_rate_limiter: - results = await asyncio.wait_for( - self._catch(self.resolver.resolve_address, query, **kwargs), self.timeout + 10 - ) - if cache_result: + if use_cache: + results = self._dns_cache.get(dns_cache_hash, []) + if not results: + results = await self._catch(self.resolver.resolve_address, query, **kwargs) + if use_cache: self._dns_cache[dns_cache_hash] = results break except ( @@ -264,6 +263,10 @@ async def _resolve_ip(self, query, **kwargs): if tries_left > 0: retry_num = (retries + 2) - tries_left self.debug(f"Retrying (#{retry_num}) {query} with kwargs={kwargs}") + + if results: + self._last_dns_success = time.time() + self.debug(f"Results for {query} with kwargs={kwargs}: {results}") return results, errors @@ -356,7 +359,7 @@ async def resolve_event(self, event, minimal=False): types = ("A", "AAAA") if types: - tasks = [self.resolve_raw(event_host, type=t, cache_result=True) for t in types] + tasks = [self.resolve_raw(event_host, type=t, use_cache=True) for t in types] async for task in as_completed(tasks): resolved_raw, errors = await task for rdtype, e in errors: @@ -493,13 +496,15 @@ async def _catch(self, callback, *args, **kwargs): except dns.resolver.NoNameservers: raise except (dns.exception.Timeout, dns.resolver.LifetimeTimeout, TimeoutError): - log.verbose(f"DNS query with args={args}, kwargs={kwargs} timed out after {self.timeout} seconds") + log.debug(f"DNS query with args={args}, kwargs={kwargs} timed out after {self.timeout} seconds") raise except dns.exception.DNSException as e: self.debug(f"{e} (args={args}, kwargs={kwargs})") except Exception as e: log.warning(f"Error in {callback.__qualname__}() with args={args}, kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) + except asyncio.CancelledError: + return [] return [] async def is_wildcard(self, query, ips=None, rdtype=None): @@ -549,7 +554,7 @@ async def is_wildcard(self, query, ips=None, rdtype=None): if ips is None: # then resolve the query for all rdtypes base_query_tasks = { - t: asyncio.create_task(self.resolve_raw(query, type=t, cache_result=True)) for t in rdtypes_to_check + t: asyncio.create_task(self.resolve_raw(query, type=t, use_cache=True)) for t in rdtypes_to_check } for _rdtype, task in base_query_tasks.items(): raw_results, errors = await task @@ -656,7 +661,7 @@ async def is_wildcard_domain(self, domain, log_info=False): # continue for _ in range(self.wildcard_tests): rand_query = f"{rand_string(digits=False, length=10)}.{host}" - wildcard_tasks[rdtype].append(self.resolve(rand_query, type=rdtype, cache_result=False)) + wildcard_tasks[rdtype].append(self.resolve(rand_query, type=rdtype, use_cache=False)) # combine the random results is_wildcard = False @@ -685,6 +690,26 @@ async def is_wildcard_domain(self, domain, log_info=False): return wildcard_domain_results + async def _connectivity_check(self, interval=5): + """ + Used to periodically check whether the scan has an internet connection + """ + if self._last_dns_success is not None: + if time.time() - self._last_dns_success < interval: + return True + dns_server_working = [] + async with self._dns_connectivity_lock: + with suppress(Exception): + dns_server_working = await self._catch(self.resolver.resolve, "www.google.com", rdtype="A") + if dns_server_working: + self._last_dns_success = time.time() + return True + if time.time() - self._last_connectivity_warning > interval: + log.warning(f"DNS queries are failing, please check your internet connection") + self._last_connectivity_warning = time.time() + self._errors.clear() + return False + def debug(self, *args, **kwargs): if self._debug: log.debug(*args, **kwargs) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 80da115d6..009874037 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -91,6 +91,7 @@ "gay", "gentle", "giddy", + "glowering", "glutinous", "gothic", "grievous", diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 21fd1082f..63edebcc5 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -30,6 +30,8 @@ def extract_cookies(self, *args, **kwargs): class BBOTAsyncClient(httpx.AsyncClient): def __init__(self, *args, **kwargs): self._bbot_scan = kwargs.pop("_bbot_scan") + web_requests_per_second = self._bbot_scan.config.get("web_requests_per_second", 100) + self._rate_limiter = RateLimiter(web_requests_per_second, "Web") http_debug = self._bbot_scan.config.get("http_debug", None) if http_debug: @@ -59,6 +61,10 @@ def __init__(self, *args, **kwargs): if not self._persist_cookies: self._cookies = DummyCookies() + async def request(self, *args, **kwargs): + async with self._rate_limiter: + return await super().request(*args, **kwargs) + def build_request(self, *args, **kwargs): request = super().build_request(*args, **kwargs) # add custom headers if the URL is in-scope @@ -89,8 +95,6 @@ def __init__(self, parent_helper): self.parent_helper = parent_helper self.http_debug = self.parent_helper.config.get("http_debug", False) self.ssl_verify = self.parent_helper.config.get("ssl_verify", False) - self.web_requests_per_second = self.parent_helper.config.get("web_requests_per_second", 50) - self.web_rate_limiter = RateLimiter(self.web_requests_per_second, "Web") self.web_client = self.AsyncClient(persist_cookies=False) def AsyncClient(self, *args, **kwargs): @@ -135,8 +139,7 @@ async def request(self, *args, **kwargs): if self.http_debug: logstr = f"Web request: {str(args)}, {str(kwargs)}" log.debug(logstr) - async with self.web_rate_limiter: - response = await client.request(*args, **kwargs) + response = await client.request(*args, **kwargs) if self.http_debug: log.debug( f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}" diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 314e288b7..2ec1031af 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -185,7 +185,7 @@ async def massdns(self, domain, subdomains): # everything checks out self.verbose(f"Resolving batch of {len(results):,} results") - resolved = dict([l async for l in self.helpers.resolve_batch(results, type=("A", "CNAME"), cache_result=True)]) + resolved = dict([l async for l in self.helpers.resolve_batch(results, type=("A", "CNAME"))]) resolved = {k: v for k, v in resolved.items() if v} for hostname in resolved: self.add_found(hostname) diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index e2ea53691..09c3a4d2b 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -53,12 +53,19 @@ async def test_dns(bbot_scanner, bbot_config): assert any([helpers.is_subdomain(h) for h in resolved]) # dns cache + helpers.dns._dns_cache.clear() assert hash(f"8.8.8.8:PTR") not in helpers.dns._dns_cache assert hash(f"dns.google:A") not in helpers.dns._dns_cache assert hash(f"dns.google:AAAA") not in helpers.dns._dns_cache - await helpers.resolve("8.8.8.8", cache_result=True) + await helpers.resolve("8.8.8.8", use_cache=False) + await helpers.resolve("dns.google", use_cache=False) + assert hash(f"8.8.8.8:PTR") not in helpers.dns._dns_cache + assert hash(f"dns.google:A") not in helpers.dns._dns_cache + assert hash(f"dns.google:AAAA") not in helpers.dns._dns_cache + + await helpers.resolve("8.8.8.8") assert hash(f"8.8.8.8:PTR") in helpers.dns._dns_cache - await helpers.resolve("dns.google", cache_result=True) + await helpers.resolve("dns.google") assert hash(f"dns.google:A") in helpers.dns._dns_cache assert hash(f"dns.google:AAAA") in helpers.dns._dns_cache diff --git a/poetry.lock b/poetry.lock index 546d72867..83cccd904 100644 --- a/poetry.lock +++ b/poetry.lock @@ -530,14 +530,12 @@ optimize = ["orjson"] [[package]] name = "dnspython" -version = "2.4.1" +version = "2.5.0" description = "DNS toolkit" optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "dnspython-2.4.1-py3-none-any.whl", hash = "sha256:5b7488477388b8c0b70a8ce93b227c5603bc7b77f1565afe8e729c36c51447d7"}, - {file = "dnspython-2.4.1.tar.gz", hash = "sha256:c33971c79af5be968bb897e95c2448e11a645ee84d93b265ce0b7aabe5dfdca8"}, -] +python-versions = "^3.8" +files = [] +develop = false [package.extras] dnssec = ["cryptography (>=2.6,<42.0)"] @@ -547,6 +545,12 @@ idna = ["idna (>=2.1,<4.0)"] trio = ["trio (>=0.14,<0.23)"] wmi = ["wmi (>=1.5.1,<2.0.0)"] +[package.source] +type = "git" +url = "https://github.com/rthalley/dnspython" +reference = "9d0262a92c0cf105d12f524f4f103abe913458ae" +resolved_reference = "9d0262a92c0cf105d12f524f4f103abe913458ae" + [[package]] name = "docutils" version = "0.20.1" @@ -1779,4 +1783,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "4e1c3bdd376ba5b7b626e58adb54624fe46c37763304aff23881f97a3524936c" +content-hash = "6407989e2f7626a030836937e51601d79594d3396a2590e41d6dca9ce8473735" diff --git a/pyproject.toml b/pyproject.toml index 52f9624e3..85091b840 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ omegaconf = "^2.3.0" tldextract = "^3.4.0" psutil = "^5.9.4" wordninja = "^2.0.0" -dnspython = "^2.3.0" pydantic = "^1.10.6" ansible-runner = "^2.3.2" deepdiff = "^6.2.3" @@ -33,6 +32,7 @@ beautifulsoup4 = "^4.12.2" lxml = "^4.9.2" httpx = {extras = ["http2"], version = "^0.24.1"} anyio = "==4.0.0rc1" +dnspython = {git = "https://github.com/rthalley/dnspython", rev = "9d0262a92c0cf105d12f524f4f103abe913458ae"} [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" From 49f85abe0dedaa586595430cd71d5eb3d2a91f9a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 7 Aug 2023 15:13:13 -0400 Subject: [PATCH 02/29] small bugfixes --- bbot/cli.py | 2 +- bbot/core/helpers/dns.py | 2 -- bbot/modules/bypass403.py | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 4f3d70fe2..df8cfb9fa 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -132,7 +132,7 @@ async def _main(): # if none of the output modules provided on the command line are consoleable, don't turn off the defaults. Instead, just add the one specified to the defaults. if not any(o in consoleable_output_modules for o in output_modules): - output_modules += default_output_modules + output_modules.update(default_output_modules) scanner = Scanner( *options.targets, diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 914144e91..2e653fade 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -503,8 +503,6 @@ async def _catch(self, callback, *args, **kwargs): except Exception as e: log.warning(f"Error in {callback.__qualname__}() with args={args}, kwargs={kwargs}: {e}") log.trace(traceback.format_exc()) - except asyncio.CancelledError: - return [] return [] async def is_wildcard(self, query, ips=None, rdtype=None): diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py index 68500a5fd..98cffd58a 100644 --- a/bbot/modules/bypass403.py +++ b/bbot/modules/bypass403.py @@ -125,6 +125,8 @@ async def handle_event(self, event): collapse_threshold = 10 results = await self.do_checks(compare_helper, event, collapse_threshold) + if results is None: + return if len(results) > collapse_threshold: self.emit_event( { From b54ffb3408b92947fc462da234177c53ce6da375 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 7 Aug 2023 15:30:17 -0400 Subject: [PATCH 03/29] defcon badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e11c5312..61ef06cbe 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ### OSINT automation for hackers. -[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Pypi Downloads](https://img.shields.io/pypi/dm/bbot)](https://pypi.org/project/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) +[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Pypi Downloads](https://img.shields.io/pypi/dm/bbot)](https://pypi.org/project/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) BBOT is a modular, recursive OSINT framework that can execute the entire OSINT workflow in a single command. From 6613b369814921b5f9b4718684e6f7933c79f6e1 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 7 Aug 2023 15:59:51 -0400 Subject: [PATCH 04/29] fix bug with output modules' watched_events --- bbot/modules/output/base.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index 7f5b02b33..204a8f44a 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -10,6 +10,23 @@ class BaseOutputModule(BaseModule): _stats_exclude = True def _event_precheck(self, event): + # special signal event types + if event.type in ("FINISHED",): + return True, "its type is FINISHED" + if self.errored: + return False, f"module is in error state" + # exclude non-watched types + if not any(t in self.get_watched_events() for t in ("*", event.type)): + return False, "its type is not in watched_events" + if self.target_only: + if "target" not in event.tags: + return False, "it did not meet target_only filter criteria" + # exclude certain URLs (e.g. javascript): + if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: + return False, "its extension was listed in url_extension_httpx_only" + # forced events like intermediary links in a DNS resolution chain + + # output module specific stuff # omitted events such as HTTP_RESPONSE etc. if event._omit: return False, "_omit is True" @@ -20,7 +37,21 @@ def _event_precheck(self, event): # or events that are over our report distance if event._internal: return False, "_internal is True" - return super()._event_precheck(event) + + # if event is an IP address that was speculated from a CIDR + source_is_range = getattr(event.source, "type", "") == "IP_RANGE" + if ( + source_is_range + and event.type == "IP_ADDRESS" + and str(event.module) == "speculate" + and self.name != "speculate" + ): + # and the current module listens for both ranges and CIDRs + if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]): + # then skip the event. + # this helps avoid double-portscanning both an individual IP and its parent CIDR. + return False, "module consumes IP ranges directly" + return True, "precheck succeeded" def _prep_output_dir(self, filename): self.output_file = self.config.get("output_file", "") From de3bb53a976802c0d2d52a32806eca8eec044e78 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 7 Aug 2023 16:02:31 -0400 Subject: [PATCH 05/29] updated release history --- docs/release_history.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/release_history.md b/docs/release_history.md index 1396ca761..9bb4a8367 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,3 +1,35 @@ +## v1.1.0 +August 4, 2023 + +**New Features**: + +- Complete Asyncification +- Documentation + auto-updating pipelines +- Ability to list flags and their descriptions with `-lf` +- Fine-grained rate-limiting for HTTP and DNS + +**Improvements / Fixes**: + +- Better tests (one for each individual module, 91% test coverage) +- New and improved paramminer modules +- Misc bugfixes + +**New Modules**: + +- Git (detects exposed .git folder on websites) +- [Subdomain Center](https://www.subdomain.center/) (subdomain enumeration) +- [Columbus API](https://columbus.elmasy.com/) (subdomain enumeration) +- MySSL (subdomain enumeration) +- Sitedossier (subdomain enumeration) +- Digitorus (subdomain enumeration) +- Nmap (port scanner, more reliable than naabu) + - Naabu has been removed due to reliability issues +- NSEC (DNSSEC zone-walking for subdomain enumeration) +- OAUTH (Enumerates OAUTH / OpenID-Connect, detects sprayable endpoints) +- Azure Realm (Detects Managed/Federated Azure Tenants) +- Subdomains output module + + ## v1.0.5 March 10, 2023 From a16bbf498a94237e38fe2cd768d215234b55aa66 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 7 Aug 2023 16:24:29 -0400 Subject: [PATCH 06/29] module docs initial --- docs/{scanning => modules}/list_of_modules.md | 0 docs/modules/nuclei.md | 1 + mkdocs.yml | 5 ++++- 3 files changed, 5 insertions(+), 1 deletion(-) rename docs/{scanning => modules}/list_of_modules.md (100%) create mode 100644 docs/modules/nuclei.md diff --git a/docs/scanning/list_of_modules.md b/docs/modules/list_of_modules.md similarity index 100% rename from docs/scanning/list_of_modules.md rename to docs/modules/list_of_modules.md diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md new file mode 100644 index 000000000..f6e0339af --- /dev/null +++ b/docs/modules/nuclei.md @@ -0,0 +1 @@ +Placeholder \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index bd3fc780d..9d875630d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,7 +21,10 @@ nav: - Tips and Tricks: scanning/tips_and_tricks.md - Advanced Usage: scanning/advanced.md - Configuration: scanning/configuration.md - - List of Modules: scanning/list_of_modules.md + - Modules: + - List of Modules: modules/list_of_modules.md + - Module Docs: + modules/nuclei.md - Contribution: - How to Write a Module: contribution.md - Misc: From b41883c0fc2c6942532ee80aa6ccf05b5767a0ee Mon Sep 17 00:00:00 2001 From: Paul Mueller Date: Mon, 7 Aug 2023 16:45:11 -0400 Subject: [PATCH 07/29] Update nuclei.md --- docs/modules/nuclei.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md index f6e0339af..b4f218774 100644 --- a/docs/modules/nuclei.md +++ b/docs/modules/nuclei.md @@ -1 +1,16 @@ -Placeholder \ No newline at end of file +# Nuclei + +## Overview + +BBOT's interface with the open-source vulnerability scanner [Nuclei](https://github.com/projectdiscovery/nuclei) by Project Discovery. This is one of the ways BBOT makes it possible to go from a domain name or IP all the way to confirmed vulnerabilities, in one scan. + +* The BBOT Nuclei module ingests **[URL]** events and emits events of type **[VULNERABILITY]** or **[FINDING]** +* Vulnerabilities will inherit their severity from the Nuclei templates​ +* Nuclei templates of severity INFO will be emitted as **[FINDINGS]**​ + +## Default Behavior + +* By default, it will scan *only directory URLs*, but it will scan with ALL templates (**BE CAREFUL!**) +* Because it's so aggressive, its considered a **deadly** module. This means you need to use the flag **--allow-deadly** to turn it on + + From 2e77d592fbc33f64d0f7239eab96b77a5553d008 Mon Sep 17 00:00:00 2001 From: Paul Mueller Date: Mon, 7 Aug 2023 16:53:23 -0400 Subject: [PATCH 08/29] Update events.md --- docs/scanning/events.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/scanning/events.md b/docs/scanning/events.md index fd020b0f3..3b4cf9531 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -73,3 +73,19 @@ Below is a full list of event types along with which modules produce/consume the | WAF | 0 | 1 | | wafw00f | | WEBSCREENSHOT | 0 | 1 | | gowitness | + +## Findings Vs. Vulnerabilties + +BBOT has a sharp distinction between Findings and Vulnerabilities: + +**Vulnerability** + +* There's a higher standard for what is allowed to be a vulnerability. They should be considered **confirmed** and **actionable​** - no additional confirmation required +* They are always assigned a severity. The possible severities are: LOW, MEDIUM, HIGH, or CRITICAL​ + +**FINDING​** + +* Findings can range anywhere from "slightly interesting behavior" to "likely, but unconfirmed vulnerability"​ +* Are often false positives + +By making this separation, actionable vulnerabilities can be identified quickly in the midst of a large scan From c90f95078c7ed6fee17e92182d5a6846eace62e6 Mon Sep 17 00:00:00 2001 From: Paul Mueller Date: Mon, 7 Aug 2023 17:49:42 -0400 Subject: [PATCH 09/29] Update nuclei.md --- docs/modules/nuclei.md | 85 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md index b4f218774..c5a2eb9e9 100644 --- a/docs/modules/nuclei.md +++ b/docs/modules/nuclei.md @@ -4,13 +4,94 @@ BBOT's interface with the open-source vulnerability scanner [Nuclei](https://github.com/projectdiscovery/nuclei) by Project Discovery. This is one of the ways BBOT makes it possible to go from a domain name or IP all the way to confirmed vulnerabilities, in one scan. +![Nuclei Killchain](https://github.com/blacklanternsecurity/bbot/assets/24899338/7174c4ba-4a6e-4596-bb89-5a0c5f5abe74) + + * The BBOT Nuclei module ingests **[URL]** events and emits events of type **[VULNERABILITY]** or **[FINDING]** * Vulnerabilities will inherit their severity from the Nuclei templates​ -* Nuclei templates of severity INFO will be emitted as **[FINDINGS]**​ +* Nuclei templates of severity INFO will be emitted as **[FINDINGS]** ## Default Behavior * By default, it will scan *only directory URLs*, but it will scan with ALL templates (**BE CAREFUL!**) -* Because it's so aggressive, its considered a **deadly** module. This means you need to use the flag **--allow-deadly** to turn it on +* Because it's so aggressive, its considered a **deadly** module. This means you need to use the flag **--allow-deadly** to turn it on. + +## Configuration and Options + +The Nuclei module has many configuration options: + +| Option | Description | Default | +|----------------|--------------------------------------------------------------------------|---------| +| version | What version of Nuclei to use | 2.9.9 | +| tags | Limit Nuclei to templates w/these tags | | +| templates | Path to template file, or template directory | | +| severity | Filter based on severity field available in the template | | +| ratelimit | maximum number of requests to send per second | 150 | +| concurrency | maximum number of templates to be executed in parallel | 25 | +| mode | technology \| severe \| manual \| budget | manual | +| etags | Tags to exclude from the scan | | +| directory_only | When on, limits scan to only "directory" URLs (omit endpoints) | True | +| budget | Used in budget mode to set the number of requests which will be allotted | 1 | + +Most of these you probably will **NOT** want to change. In particular, we strongly advise against changing the version of Nuclei, as it's very likely the latest version won't work right with BBOT. + +We also do not recommend change **directory_only** mode. Because BBOT is recursive, feeding Nuclei every URL can get very out-of-hand very quickly, depending on what other modules are in use. + +### Mode ### + +The modes with the Nuclei module are generally in place to help you limit the number of templates you are scanning with, to make your scans quicker. + +#### Manual + +This is the default setting, and will use all templates. However, if you're looking to do something particular, you might pair this with some of the pass-through options shown in the next setting. + +#### Severe + +**severe** mode uses only high/critical severity templates. It also excludes the intrusive tag. This is intended to be a shortcut for times when you need to rapidly identify high severity vulnerabilities but can't afford the full scan. Because most templates are INFO, LOW, or MEDIUM, your scan will finish much faster. + +#### Technology + +This is equivalent to the Nuclei '-as' scan option. It only use templates that match detected technologies, using wappalyzer-based signatures. This can be a nice way to run a light-weight scan that still has a chance to find some good vulnerabilities. + +#### Budget + +Budget mode is unique to BBOT. ​ + +For larger scans with thousands of targets, doing a FULL Nuclei scan (1000s of Requests) for each is not realistic. ​ +As an alternative to the other modes, you can take advantage of Nuclei's "collapsible" template feature. ​ + +For only the cost of one (or more) "extra" request(s) per host, it can activate several hundred modules. These are modules which happen to look at a BaseUrl, and typically look for a specific string or other attribute. Nuclei is smart about reusing the request data when it can, and we can use this to our advantage. + +The budget parameter is the # of extra requests per host you are willing to send to "feed" Nuclei templates​ (defaults to 1). +For those times when vulnerability scanning isn't the main focus, but you want to look for easy wins.​ + +Of course, there is a rapidly diminishing return when you set he value to more than a handful. Eventually, this becomes 1 template per 1 budget value increase. However, in the 1-10 range there is a lot of value. This graphic should give you a rough visual idea of this concept. + +![Nuclei Budget Mode](https://github.com/blacklanternsecurity/bbot/assets/24899338/08a3429c-5a73-437b-84de-27c07d85a529) + + +### Nuclei pass-through options + +Most of the rest of the options are usually passed straight through to Nuclei when its executed. You can do things like set specific **tags** to include, (or exclude with **etags**), exactly how you'd do with Nuclei directly. You can also limit the templates with **severity**. + +The **ratelimit** and **concurrency** settings default to the same defaults that Nuclei does. These are relatively sane settings, but if you are in a sensitive environment it can certainly help to turn them down. + +**templates** will allow you to set your own templates directory. This can be very useful if you have your own custom templates that you want to use with BBOT. + +### Example Commands + +* Scan a SINGLE target with a basic port scan and web modules + +`COMMAND: bbot -f web-basic -m nmap nuclei --allow-deadly -t app.evilcorp.com​` + +* Scanning MULTIPLE targets + +`bbot -f web-basic -m nmap nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com​` + +* Scanning MULTIPLE targets while performing subdomain enumeration + +`bbot -f subdomain-enum web-basic -m nmap nuclei –allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com​` +* Scanning MULTIPLE targets on a BUDGET​ +`bbot -f subdomain-enum web-basic -m nmap nuclei –allow-deadly –c modules.nuclei.mode=Budget -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com​` From 6211745fa35972d4d5fb4e7c1460668a0a839e99 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 7 Aug 2023 17:54:06 -0400 Subject: [PATCH 10/29] mkdocs.yaml fix --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 9d875630d..98a10ac18 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -24,7 +24,7 @@ nav: - Modules: - List of Modules: modules/list_of_modules.md - Module Docs: - modules/nuclei.md + - Nuclei: modules/nuclei.md - Contribution: - How to Write a Module: contribution.md - Misc: From 62048319b813bb1a2d32b269981b3e11e643dcac Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 7 Aug 2023 18:01:20 -0400 Subject: [PATCH 11/29] troubleshooting update docs --- mkdocs.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 98a10ac18..a6ae600fa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,8 +23,7 @@ nav: - Configuration: scanning/configuration.md - Modules: - List of Modules: modules/list_of_modules.md - - Module Docs: - - Nuclei: modules/nuclei.md + - Nuclei: modules/nuclei.md - Contribution: - How to Write a Module: contribution.md - Misc: From d6d9728c93e90eda4d18d88f2e82c7312c58ba41 Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:19:06 -0400 Subject: [PATCH 12/29] Update nuclei.md --- docs/modules/nuclei.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md index c5a2eb9e9..6beb3f870 100644 --- a/docs/modules/nuclei.md +++ b/docs/modules/nuclei.md @@ -35,7 +35,7 @@ The Nuclei module has many configuration options: Most of these you probably will **NOT** want to change. In particular, we strongly advise against changing the version of Nuclei, as it's very likely the latest version won't work right with BBOT. -We also do not recommend change **directory_only** mode. Because BBOT is recursive, feeding Nuclei every URL can get very out-of-hand very quickly, depending on what other modules are in use. +We also do not recommend changing **directory_only** mode. Because BBOT is recursive, feeding Nuclei every URL can get very out-of-hand very quickly, depending on what other modules are in use. ### Mode ### From c2ff42c989a93e3280e759ca55e9efb8980d19d8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 13:14:17 -0400 Subject: [PATCH 13/29] links for seo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 61ef06cbe..57ea2b226 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,8 @@ If you're interested in contributing to BBOT, or just curious how it works under Special thanks to the following people who made BBOT possible: -- @TheTechromancer for creating BBOT -- @liquidsec for his extensive work on BBOT's web hacking features +- @TheTechromancer for creating [BBOT](https://github.com/blacklanternsecurity/bbot) +- @liquidsec for his extensive work on BBOT's web hacking features, including [badsecrets](https://github.com/blacklanternsecurity/badsecrets) - Steve Micallef (@smicallef) for creating Spiderfoot - @kerrymilan for his Neo4j and Ansible expertise - Aleksei Kornev (@alekseiko) for allowing us ownership of the bbot Pypi repository <3 From 1ffeefd65d2a2cedb340c64c4afc835b6b655862 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 14:55:22 -0400 Subject: [PATCH 14/29] update pyproject.toml --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 85091b840..3d6e5334d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,8 @@ license = "GPL-3.0" readme = "README.md" repository = "https://github.com/blacklanternsecurity/bbot" homepage = "https://github.com/blacklanternsecurity/bbot" +documentation = "https://www.blacklanternsecurity.com/bbot/" +keywords = ["python", "cli", "automation", "osint", "neo4j", "scanner", "python-library", "hacking", "recursion", "pentesting", "recon", "command-line-tool", "bugbounty", "subdomains", "security-tools", "subdomain-scanner", "osint-framework", "attack-surface", "subdomain-enumeration", "osint-tool"] [tool.poetry.scripts] bbot = 'bbot.cli:main' From 84701192ce82a9f6cf8b35f423a9d703faf1c81b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 14:59:57 -0400 Subject: [PATCH 15/29] link to bbot repo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57ea2b226..b7377dad9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -![bbot_banner](https://user-images.githubusercontent.com/20261699/158000235-6c1ace81-a267-4f8e-90a1-f4c16884ebac.png) +[![bbot_banner](https://user-images.githubusercontent.com/20261699/158000235-6c1ace81-a267-4f8e-90a1-f4c16884ebac.png)](https://github.com/blacklanternsecurity/bbot) # BEE·bot From 5f80313bd74ca0f5c4e70a9e060ead39d7ba6a84 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 15:04:56 -0400 Subject: [PATCH 16/29] description --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7377dad9..3823a1d99 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Pypi Downloads](https://img.shields.io/pypi/dm/bbot)](https://pypi.org/project/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA) -BBOT is a modular, recursive OSINT framework that can execute the entire OSINT workflow in a single command. +BBOT (Bighuge BLS OSINT Tool) is a modular, recursive OSINT framework that can execute the entire OSINT workflow in a single command. BBOT is inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot) but takes it to the next level with features like multi-target scans, lightning-fast asyncio performance, and NLP-powered subdomain mutations. It offers a wide range of functionality, including subdomain enumeration, port scanning, web screenshots, vulnerability scanning, and much more. From 66201c0d2c2fe4e06c794ad2420e94cf92c74205 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 15:06:56 -0400 Subject: [PATCH 17/29] update authors --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3d6e5334d..f0bf5d7de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,10 @@ name = "bbot" version = "1.0.3" description = "OSINT automation for hackers." -authors = ["TheTechromancer"] +authors = [ + "TheTechromancer", + "Paul Mueller", +] license = "GPL-3.0" readme = "README.md" repository = "https://github.com/blacklanternsecurity/bbot" From 2f099d87ddbab1d1ebeb9ba679abe62743e64584 Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:11:57 -0400 Subject: [PATCH 18/29] Update events.md --- docs/scanning/events.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 3b4cf9531..3871d9839 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -78,7 +78,7 @@ Below is a full list of event types along with which modules produce/consume the BBOT has a sharp distinction between Findings and Vulnerabilities: -**Vulnerability** +**VULNERABILITY** * There's a higher standard for what is allowed to be a vulnerability. They should be considered **confirmed** and **actionable​** - no additional confirmation required * They are always assigned a severity. The possible severities are: LOW, MEDIUM, HIGH, or CRITICAL​ From c91c198d44131483bb0a8a170c1844773d8c6bde Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 15:16:27 -0400 Subject: [PATCH 19/29] classifiers --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f0bf5d7de..ccf1ecfa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,12 @@ repository = "https://github.com/blacklanternsecurity/bbot" homepage = "https://github.com/blacklanternsecurity/bbot" documentation = "https://www.blacklanternsecurity.com/bbot/" keywords = ["python", "cli", "automation", "osint", "neo4j", "scanner", "python-library", "hacking", "recursion", "pentesting", "recon", "command-line-tool", "bugbounty", "subdomains", "security-tools", "subdomain-scanner", "osint-framework", "attack-surface", "subdomain-enumeration", "osint-tool"] +classifiers = [ + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Programming Language :: Python :: 3", + "Operating System :: POSIX :: Linux", + "Topic :: Security", +] [tool.poetry.scripts] bbot = 'bbot.cli:main' From 50f1b09e6eb8659656df5ce704f4d1bf87473240 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 15:16:55 -0400 Subject: [PATCH 20/29] update classifiers --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ccf1ecfa9..bb8487182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,6 @@ homepage = "https://github.com/blacklanternsecurity/bbot" documentation = "https://www.blacklanternsecurity.com/bbot/" keywords = ["python", "cli", "automation", "osint", "neo4j", "scanner", "python-library", "hacking", "recursion", "pentesting", "recon", "command-line-tool", "bugbounty", "subdomains", "security-tools", "subdomain-scanner", "osint-framework", "attack-surface", "subdomain-enumeration", "osint-tool"] classifiers = [ - "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Programming Language :: Python :: 3", "Operating System :: POSIX :: Linux", "Topic :: Security", ] From 0aa551929f1be7bcc7a6a2aef7b3d3623f4ca117 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 15:19:13 -0400 Subject: [PATCH 21/29] additional URLs --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index bb8487182..0c8785ece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,10 @@ classifiers = [ "Topic :: Security", ] +[tool.poetry.urls] +"Discord" = "https://discord.com/invite/PZqkgxu5SA" +"Docker Hub" = "https://hub.docker.com/r/blacklanternsecurity/bbot" + [tool.poetry.scripts] bbot = 'bbot.cli:main' From b70349201c10f0b7c7da0123702a85f477141d38 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 8 Aug 2023 15:53:30 -0400 Subject: [PATCH 22/29] fix broken blogpost link --- docs/comparison.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/comparison.md b/docs/comparison.md index 4ce8b4016..085e9907c 100644 --- a/docs/comparison.md +++ b/docs/comparison.md @@ -12,4 +12,4 @@ Thanks to BBOT's recursive nature (and its `massdns` module with its NLP-powered ![runtimes](https://github.com/blacklanternsecurity/bbot/assets/20261699/66cafb5f-045b-4d88-9ffa-7542b3dada4f) -For a detailed analysis of this data, please see [Subdomain Enumeration Tool Face-Off](https://blog.blacklanternsecurity.com/p/subdomain-enumeration-tool-face-off-2023-edition) +For a detailed analysis of this data, please see [Subdomain Enumeration Tool Face-Off](https://blog.blacklanternsecurity.com/p/subdomain-enumeration-tool-face-off-4e5) From 2880a0ae30641cc26637d3d9e046ab0ee50c143b Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Tue, 8 Aug 2023 15:57:28 -0400 Subject: [PATCH 23/29] Update index.md --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 4309c6788..ac0017212 100644 --- a/docs/index.md +++ b/docs/index.md @@ -113,6 +113,6 @@ Or on the command-line: bbot -t evilcorp.com -f subdomain-enum -c modules.shodan_dns.api_key=deadbeef modules.virustotal.api_key=cafebabe ``` -For more information, see [Configuration](./scanning/configuration/). For a full list of modules, including which ones require API keys, see [List of Modules](./scanning/list_of_modules/). +For more information, see [Configuration](./scanning/configuration/). For a full list of modules, including which ones require API keys, see [List of Modules](./modules/list_of_modules/). [Next Up: Scanning -->](./scanning/){ .md-button .md-button--primary } From d1de777f16b95ceec14f270ec584985cf32d85e4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 9 Aug 2023 22:54:23 -0400 Subject: [PATCH 24/29] update dnspython --- poetry.lock | 18 +++++++----------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index 83cccd904..964085f21 100644 --- a/poetry.lock +++ b/poetry.lock @@ -530,12 +530,14 @@ optimize = ["orjson"] [[package]] name = "dnspython" -version = "2.5.0" +version = "2.4.2" description = "DNS toolkit" optional = false -python-versions = "^3.8" -files = [] -develop = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "dnspython-2.4.2-py3-none-any.whl", hash = "sha256:57c6fbaaeaaf39c891292012060beb141791735dbb4004798328fc2c467402d8"}, + {file = "dnspython-2.4.2.tar.gz", hash = "sha256:8dcfae8c7460a2f84b4072e26f1c9f4101ca20c071649cb7c34e8b6a93d58984"}, +] [package.extras] dnssec = ["cryptography (>=2.6,<42.0)"] @@ -545,12 +547,6 @@ idna = ["idna (>=2.1,<4.0)"] trio = ["trio (>=0.14,<0.23)"] wmi = ["wmi (>=1.5.1,<2.0.0)"] -[package.source] -type = "git" -url = "https://github.com/rthalley/dnspython" -reference = "9d0262a92c0cf105d12f524f4f103abe913458ae" -resolved_reference = "9d0262a92c0cf105d12f524f4f103abe913458ae" - [[package]] name = "docutils" version = "0.20.1" @@ -1783,4 +1779,4 @@ xmltodict = ">=0.12.0,<0.13.0" [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "6407989e2f7626a030836937e51601d79594d3396a2590e41d6dca9ce8473735" +content-hash = "f466d106f02b1d01755bc0caa87eaeb20c3ccb9ef598bc0cb15a90c2738340c0" diff --git a/pyproject.toml b/pyproject.toml index 0c8785ece..f7ed520ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ beautifulsoup4 = "^4.12.2" lxml = "^4.9.2" httpx = {extras = ["http2"], version = "^0.24.1"} anyio = "==4.0.0rc1" -dnspython = {git = "https://github.com/rthalley/dnspython", rev = "9d0262a92c0cf105d12f524f4f103abe913458ae"} +dnspython = "^2.4.2" [tool.poetry.group.dev.dependencies] pytest = "^7.2.2" From a465493e614efa0dbae8bb75c816c90cccd8792b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 9 Aug 2023 23:05:04 -0400 Subject: [PATCH 25/29] better results logging for dns (debug) --- bbot/core/helpers/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index 2e653fade..d40b296cc 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -118,6 +118,7 @@ async def resolve(self, query, **kwargs): log.trace(traceback.format_exc()) raise + self.debug(f"Results for {query} with kwargs={kwargs}: {results}") return results async def resolve_raw(self, query, **kwargs): @@ -267,7 +268,6 @@ async def _resolve_ip(self, query, **kwargs): if results: self._last_dns_success = time.time() - self.debug(f"Results for {query} with kwargs={kwargs}: {results}") return results, errors async def handle_wildcard_event(self, event, children): From e60534ce81b7c9d6caa73cc95b0a74e26c75f6e6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 9 Aug 2023 23:37:14 -0400 Subject: [PATCH 26/29] verbosify some dns log output --- bbot/core/helpers/dns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py index d40b296cc..1d5fb7bef 100644 --- a/bbot/core/helpers/dns.py +++ b/bbot/core/helpers/dns.py @@ -190,7 +190,7 @@ async def _resolve_hostname(self, query, **kwargs): f'Aborting query "{query}" because failed {rdtype} queries for "{parent}" ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' ) if parent_hash not in self._dns_warnings: - log.info( + log.verbose( f'Aborting future {rdtype} queries to "{parent}" because error count ({error_count:,}) exceeded abort threshold ({self.abort_threshold:,})' ) self._dns_warnings.add(parent_hash) From b476c872d8daab1c57ec1ce851b18652bc2c7072 Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Thu, 10 Aug 2023 10:27:48 -0700 Subject: [PATCH 27/29] increase dns abort threshold --- bbot/defaults.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 4e69371ba..5ed85b31d 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -75,7 +75,7 @@ dns_wildcard_ignore: [] dns_wildcard_tests: 10 # Skip DNS requests for a certain domain and rdtype after encountering this many timeouts or SERVFAILs # This helps prevent faulty DNS servers from hanging up the scan -dns_abort_threshold: 10 +dns_abort_threshold: 50 # Don't show PTR records containing IP addresses dns_filter_ptrs: true # Enable/disable debug messages for dns queries From b4d6af54b32f9fc7aa7e298d9b0dff70665e7c40 Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Thu, 10 Aug 2023 17:34:24 +0000 Subject: [PATCH 28/29] Refresh module docs --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3823a1d99..77ed65a57 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,9 @@ asyncio.run(main()) - [Tips and Tricks](https://www.blacklanternsecurity.com/bbot/scanning/tips_and_tricks) - [Advanced Usage](https://www.blacklanternsecurity.com/bbot/scanning/advanced) - [Configuration](https://www.blacklanternsecurity.com/bbot/scanning/configuration) - - [List of Modules](https://www.blacklanternsecurity.com/bbot/scanning/list_of_modules) +- **Modules** + - [List of Modules](https://www.blacklanternsecurity.com/bbot/modules/list_of_modules) + - [Nuclei](https://www.blacklanternsecurity.com/bbot/modules/nuclei) - **Contribution** - [How to Write a Module](https://www.blacklanternsecurity.com/bbot/contribution) - **Misc** From 0161a657a3caf682ea0b676877ae8e421c481948 Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Thu, 10 Aug 2023 18:06:03 +0000 Subject: [PATCH 29/29] Refresh module docs --- docs/scanning/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 8a641ea15..8a1a46508 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -132,7 +132,7 @@ dns_wildcard_ignore: [] dns_wildcard_tests: 10 # Skip DNS requests for a certain domain and rdtype after encountering this many timeouts or SERVFAILs # This helps prevent faulty DNS servers from hanging up the scan -dns_abort_threshold: 10 +dns_abort_threshold: 50 # Don't show PTR records containing IP addresses dns_filter_ptrs: true # Enable/disable debug messages for dns queries