From 7d0ff2417d99f6e03dfed9a64079a9cd44f20252 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 11 Oct 2023 15:10:41 -0400 Subject: [PATCH 01/49] filedownload module --- bbot/modules/filedownload.py | 165 +++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 bbot/modules/filedownload.py diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py new file mode 100644 index 000000000..8b61dbdd8 --- /dev/null +++ b/bbot/modules/filedownload.py @@ -0,0 +1,165 @@ +import json +from pathlib import Path + +from bbot.modules.base import BaseModule + + +class filedownload(BaseModule): + """ + Watch for common filetypes and download them. + + Capable of identifying interesting files even if the extension is not in the URL. + E.g. if a PDF is being served at https://evilcorp.com/mypdf, it will still be downloaded and given the proper extension. + """ + + watched_events = ["URL_UNVERIFIED", "HTTP_RESPONSE"] + produced_events = [] + flags = ["active", "safe"] + meta = {"description": "Download common filetypes such as PDF, DOCX, PPTX, etc."} + options = { + "extensions": [ + "bak", # Backup File + "bash", # Bash Script or Configuration + "bashrc", # Bash Script or Configuration + "conf", # Configuration File + "cfg", # Configuration File + "crt", # Certificate File + "csv", # Comma Separated Values File + "db", # SQLite Database File + "sqlite", # SQLite Database File + "doc", # Microsoft Word Document (Old Format) + "docx", # Microsoft Word Document + "exe", # Windows PE executable + "ica", # Citrix Independent Computing Architecture File + "indd", # Adobe InDesign Document + "ini", # Initialization File + "jar", # Java Archive + "key", # Private Key File + "pub", # Public Key File + "log", # Log File + "markdown", # Markdown File + "md", # Markdown File + "msi", # Windows setup file + "odg", # OpenDocument Graphics (LibreOffice, OpenOffice) + "odp", # OpenDocument Presentation (LibreOffice, OpenOffice) + "ods", # OpenDocument Spreadsheet (LibreOffice, OpenOffice) + "odt", # OpenDocument Text (LibreOffice, OpenOffice) + "pdf", # Adobe Portable Document Format + "pem", # Privacy Enhanced Mail (SSL certificate) + "png", # Portable Network Graphics Image + "pps", # Microsoft PowerPoint Slideshow (Old Format) + "ppsx", # Microsoft PowerPoint Slideshow + "ppt", # Microsoft PowerPoint Presentation (Old Format) + "pptx", # Microsoft PowerPoint Presentation + "ps1", # PowerShell Script + "raw", # Raw Image File Format + "rdp", # Remote Desktop Protocol File + "sh", # Shell Script + "sql", # SQL Database Dump + "swp", # Swap File (temporary file, often Vim) + "sxw", # OpenOffice.org Writer document + "tar", # Tar Archive + "tar.gz", # Gzip-Compressed Tar Archive + "zip", # Zip Archive + "txt", # Plain Text Document + "vbs", # Visual Basic Script + "wpd", # WordPerfect Document + "xls", # Microsoft Excel Spreadsheet (Old Format) + "xlsx", # Microsoft Excel Spreadsheet + "xml", # eXtensible Markup Language File + "yml", # YAML Ain't Markup Language + "yaml", # YAML Ain't Markup Language + ], + "max_filesize": "10MB", + } + options_desc = { + "extensions": "File extensions to download", + "max_filesize": "Cancel download if filesize is greater than this size", + } + + scope_distance_modifier = 1 + + async def setup(self): + self.extensions = list(set([e.lower().strip(".") for e in self.options.get("extensions", [])])) + self.max_filesize = self.options.get("max_filesize", "10MB") + self.download_dir = self.scan.home / "filedownload" + self.helpers.mkdir(self.download_dir) + self.files_downloaded = set() + self.mime_db_file = await self.helpers.wordlist( + "https://raw.githubusercontent.com/jshttp/mime-db/master/db.json" + ) + self.mime_db = {} + with open(self.mime_db_file) as f: + mime_db = json.load(f) + for content_type, attrs in mime_db.items(): + if "extensions" in attrs and attrs["extensions"]: + self.mime_db[content_type] = attrs["extensions"][0].lower() + return True + + async def filter_event(self, event): + # accept file download requests from other modules + if "filedownload" in event.tags: + return True + h = self.hash_event(event) + if h in self.files_downloaded: + return False, f"Already processed {event}" + return True + + def hash_event(self, event): + if event.type == "HTTP_RESPONSE": + return hash(event.data["url"]) + return hash(event.data) + + async def handle_event(self, event): + if event.type == "URL_UNVERIFIED": + url_lower = event.data.lower() + if any(url_lower.endswith(f".{e}") for e in self.extensions): + await self.download_file(event.data) + elif event.type == "HTTP_RESPONSE": + content_type = event.data["header"].get("content_type", "") + if content_type: + url = event.data["url"] + await self.download_file(url, content_type=content_type) + + async def download_file(self, url, content_type=None): + orig_filename, file_destination, base_url = self.make_filename(url, content_type=content_type) + if orig_filename is None: + return + result = await self.helpers.download(url, warn=False, filename=file_destination, max_size=self.max_filesize) + if result: + self.info(f'Found "{orig_filename}" at "{base_url}", downloaded to {file_destination}') + self.files_downloaded.add(hash(url)) + + def make_filename(self, url, content_type=None): + # first, try to determine original filename + parsed_url = self.helpers.urlparse(url) + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + url_path = parsed_url.path.strip("/") + # try to get extension from URL path + extension = Path(url_path).suffix.strip(".").lower() + if extension: + url_stem = url.rsplit(".", 1)[0] + else: + url_stem = str(url) + filename = f"{self.helpers.make_date()}_{self.helpers.tagify(url_stem)}" + if not url_path: + url_path = "unknown" + filename = f"{filename}-{url_path}" + # if that fails, try to get it from content type + if not extension: + if content_type and content_type in self.mime_db: + extension = self.mime_db[content_type] + + if (not extension) or (extension not in self.extensions): + self.debug(f'Extension "{extension}" at url "{url}" not in list of watched extensions.') + return None, None, None + + orig_filename = Path(url_path).stem + if extension: + filename = f"{filename}.{extension}" + orig_filename = f"{orig_filename}.{extension}" + return orig_filename, self.download_dir / filename, base_url + + async def report(self): + if self.files_downloaded: + self.success(f"Downloaded {len(self.files_downloaded):,} file(s) to {self.download_dir}") From af70529cab261696e1027660dc2c572b60c6e0f3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 11 Oct 2023 15:21:06 -0400 Subject: [PATCH 02/49] filedownload web helpers --- bbot/core/helpers/web.py | 82 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 5ac1cd82a..7dc6b45ca 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -7,6 +7,7 @@ import traceback from pathlib import Path from bs4 import BeautifulSoup +from contextlib import asynccontextmanager from httpx._models import Cookies @@ -272,9 +273,11 @@ async def download(self, url, **kwargs): url (str): The URL of the file to download. filename (str, optional): The filename to save the downloaded file as. If not provided, will generate based on URL. + max_size (str or int): Maximum filesize as a string ("5MB") or integer in bytes. cache_hrs (float, optional): The number of hours to cache the downloaded file. A negative value disables caching. Defaults to -1. method (str, optional): The HTTP method to use for the request, defaults to 'GET'. + raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False. **kwargs: Additional keyword arguments to pass to the httpx request. Returns: @@ -285,7 +288,15 @@ async def download(self, url, **kwargs): """ success = False filename = kwargs.pop("filename", self.parent_helper.cache_filename(url)) + follow_redirects = kwargs.pop("follow_redirects", True) + max_size = kwargs.pop("max_size", None) + warn = kwargs.pop("warn", True) + raise_error = kwargs.pop("raise_error", False) + if max_size is not None: + max_size = self.parent_helper.human_to_bytes(max_size) cache_hrs = float(kwargs.pop("cache_hrs", -1)) + total_size = 0 + chunk_size = 8192 log.debug(f"Downloading file from {url} with cache_hrs={cache_hrs}") if cache_hrs > 0 and self.parent_helper.is_cached(url): log.debug(f"{url} is cached at {self.parent_helper.cache_filename(url)}") @@ -293,20 +304,32 @@ async def download(self, url, **kwargs): else: # kwargs["raise_error"] = True # kwargs["stream"] = True + kwargs["follow_redirects"] = follow_redirects if not "method" in kwargs: kwargs["method"] = "GET" try: - async with self.AsyncClient().stream(url=url, **kwargs) as response: + async with self._acatch(url, raise_error), self.AsyncClient().stream(url=url, **kwargs) as response: status_code = getattr(response, "status_code", 0) log.debug(f"Download result: HTTP {status_code}") if status_code != 0: response.raise_for_status() with open(filename, "wb") as f: - async for chunk in response.aiter_bytes(chunk_size=8192): + agen = response.aiter_bytes(chunk_size=chunk_size) + async for chunk in agen: + if max_size is not None and total_size + chunk_size > max_size: + log.verbose( + f"Filesize of {url} exceeds {self.parent_helper.bytes_to_human(max_size)}, file will be truncated" + ) + agen.aclose() + break + total_size += chunk_size f.write(chunk) success = True except httpx.HTTPError as e: - log.warning(f"Failed to download {url}: {e}") + log_fn = log.verbose + if warn: + log_fn = log.warning + log_fn(f"Failed to download {url}: {e}") return if success: @@ -574,6 +597,59 @@ def is_spider_danger(self, source_event, url): return True return False + def ssl_context_noverify(self): + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3 + ssl_context.set_ciphers("ALL:@SECLEVEL=0") + ssl_context.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option + return ssl_context + + @asynccontextmanager + async def _acatch(self, url, raise_error): + """ + Asynchronous context manager to handle various httpx errors during a request. + + Yields: + None + + Note: + This function is internal and should generally not be used directly. + `url`, `args`, `kwargs`, and `raise_error` should be in the same context as this function. + """ + try: + yield + except httpx.TimeoutException: + log.verbose(f"HTTP timeout to URL: {url}") + if raise_error: + raise + except httpx.ConnectError: + log.verbose(f"HTTP connect failed to URL: {url}") + if raise_error: + raise + except httpx.RequestError as e: + log.trace(f"Error with request to URL: {url}: {e}") + log.trace(traceback.format_exc()) + if raise_error: + raise + except ssl.SSLError as e: + msg = f"SSL error with request to URL: {url}: {e}" + log.trace(msg) + log.trace(traceback.format_exc()) + if raise_error: + raise httpx.RequestError(msg) + except anyio.EndOfStream as e: + msg = f"AnyIO error with request to URL: {url}: {e}" + log.trace(msg) + log.trace(traceback.format_exc()) + if raise_error: + raise httpx.RequestError(msg) + except BaseException as e: + log.trace(f"Unhandled exception with request to URL: {url}: {e}") + log.trace(traceback.format_exc()) + raise + user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]] pass_keywords = [re.compile(r, re.I) for r in ["pass"]] From efda63f14e5fe1e9eaaccad4d5f35dd6a18a1dec Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 11 Oct 2023 15:21:51 -0400 Subject: [PATCH 03/49] filedownload module tests --- .../module_tests/test_module_filedownload.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 bbot/test/test_step_2/module_tests/test_module_filedownload.py diff --git a/bbot/test/test_step_2/module_tests/test_module_filedownload.py b/bbot/test/test_step_2/module_tests/test_module_filedownload.py new file mode 100644 index 000000000..e4471d159 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_filedownload.py @@ -0,0 +1,68 @@ +from .base import ModuleTestBase + + +class TestFileDownload(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["filedownload", "httpx", "excavate", "speculate"] + config_overrides = {"web_spider_distance": 2, "web_spider_depth": 2} + + pdf_data = """%PDF-1. +1 0 obj<>endobj +2 0 obj<>endobj +3 0 obj<>endobj +trailer <>""" + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url="https://raw.githubusercontent.com/jshttp/mime-db/master/db.json", + json={ + "application/pdf": {"source": "iana", "compressible": False, "extensions": ["pdf"]}, + }, + ) + + async def setup_after_prep(self, module_test): + module_test.set_expect_requests( + dict(uri="/"), + dict( + response_data='' + ), + ) + module_test.set_expect_requests( + dict(uri="/Test_File.txt"), + dict( + response_data="juicy stuff", + ), + ) + module_test.set_expect_requests( + dict(uri="/Test_PDF"), + dict(response_data=self.pdf_data, headers={"Content-Type": "application/pdf"}), + ) + module_test.set_expect_requests( + dict(uri="/test.html"), + dict(response_data="", headers={"Content-Type": "text/html"}), + ) + module_test.set_expect_requests( + dict(uri="/test2"), + dict(response_data="", headers={"Content-Type": "text/html"}), + ) + + def check(self, module_test, events): + download_dir = module_test.scan.home / "filedownload" + + # text file + text_files = list(download_dir.glob("*test-file.txt")) + assert len(text_files) == 1, f"No text file found at {download_dir}" + file = text_files[0] + assert file.is_file(), f"File not found at {file}" + assert open(file).read() == "juicy stuff", f"File at {file} does not contain the correct content" + + # PDF file (no extension) + pdf_files = list(download_dir.glob("*test-pdf.pdf")) + assert len(pdf_files) == 1, f"No PDF file found at {download_dir}" + file = pdf_files[0] + assert file.is_file(), f"File not found at {file}" + assert open(file).read() == self.pdf_data, f"File at {file} does not contain the correct content" + + # we don't want html files + html_files = list(download_dir.glob("*.html")) + assert len(html_files) == 0, "HTML files were erroneously downloaded" From 70c00c514d2de652df0d48ccd488b096255351c7 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 11 Oct 2023 15:26:17 -0400 Subject: [PATCH 04/49] finishing up filedownload helpers, ssl verification consolidation --- bbot/core/helpers/web.py | 37 +------------------------------------ bbot/modules/sslcert.py | 8 +------- 2 files changed, 2 insertions(+), 43 deletions(-) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index 7dc6b45ca..f26d4666d 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -217,7 +217,7 @@ async def request(self, *args, **kwargs): if client_kwargs: client = self.AsyncClient(**client_kwargs) - try: + async with self._acatch(url, raise_error): if self.http_debug: logstr = f"Web request: {str(args)}, {str(kwargs)}" log.debug(logstr) @@ -227,41 +227,6 @@ async def request(self, *args, **kwargs): f"Web response from {url}: {response} (Length: {len(response.content)}) headers: {response.headers}" ) return response - except httpx.PoolTimeout: - # this block exists because of this: - # https://github.com/encode/httpcore/discussions/783 - log.verbose(f"PoolTimeout to URL: {url}") - self.web_client = self.AsyncClient(persist_cookies=False) - return await self.request(*args, **kwargs) - except httpx.TimeoutException: - log.verbose(f"HTTP timeout to URL: {url}") - if raise_error: - raise - except httpx.ConnectError: - log.verbose(f"HTTP connect failed to URL: {url}") - if raise_error: - raise - except httpx.RequestError as e: - log.trace(f"Error with request to URL: {url}: {e}") - log.trace(traceback.format_exc()) - if raise_error: - raise - except ssl.SSLError as e: - msg = f"SSL error with request to URL: {url}: {e}" - log.trace(msg) - log.trace(traceback.format_exc()) - if raise_error: - raise httpx.RequestError(msg) - except anyio.EndOfStream as e: - msg = f"AnyIO error with request to URL: {url}: {e}" - log.trace(msg) - log.trace(traceback.format_exc()) - if raise_error: - raise httpx.RequestError(msg) - except BaseException as e: - log.trace(f"Unhandled exception with request to URL: {url}: {e}") - log.trace(traceback.format_exc()) - raise async def download(self, url, **kwargs): """ diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index a9f269c55..de598dd73 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -1,4 +1,3 @@ -import ssl import asyncio from OpenSSL import crypto from contextlib import suppress @@ -109,12 +108,7 @@ async def visit_host(self, host, port): # Create an SSL context try: - ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3 - ssl_context.set_ciphers("ALL:@SECLEVEL=0") - ssl_context.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option + ssl_context = self.helpers.ssl_context_noverify() except Exception as e: self.warning(f"Error creating SSL context: {e}") return [], [], (host, port) From 8f338a3c50fd5d3620aacd58423ba8b96cf99b4b Mon Sep 17 00:00:00 2001 From: SpamFaux Date: Thu, 4 May 2023 09:41:18 -0700 Subject: [PATCH 05/49] Created and added dehashed module. Built out the primary function of communicating with the API. Added PASSWORD, HASHED_PASSWORD, and USERNAME on to the base events. Troubleshooting why it's not actually emitting events. --- bbot/core/event/base.py | 8 +++++ bbot/modules/dehashed.py | 73 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 bbot/modules/dehashed.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index adfadb1ac..1b4587872 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1128,6 +1128,14 @@ def _pretty_string(self): class GEOLOCATION(BaseEvent): _always_emit = True +class PASSWORD(BaseEvent): + _always_emit = True + +class HASHED_PASSWORD(BaseEvent): + _always_emit = True + +class USERNAME(BaseEvent): + _always_emit = True class SOCIAL(DictEvent): _always_emit = True diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py new file mode 100644 index 000000000..a93029d01 --- /dev/null +++ b/bbot/modules/dehashed.py @@ -0,0 +1,73 @@ +from bbot.modules.base import BaseModule +from requests.auth import HTTPBasicAuth + +class dehashed(BaseModule): + watched_events = ["EMAIL_ADDRESS"] + produced_events = ["PASSWORD"] + flags = ["passive"] + meta = {"description": "Execute queries against dehash.com for exposed credentials", "auth_required": True} + options = {"username":"", "api_key": ""} + options_desc = {"username":"Email Address associated with your API key","api_key": "API Key"} + + base_url = "https://api.dehashed.com/search" + + def setup(self): + self.username = self.config.get("username", "") + self.api_key = self.config.get("api_key","") + self.auth = HTTPBasicAuth(self.username, self.api_key) + self.headers = { + "Accept": "application/json", + } + + self.result_passwords = {} + self.result_usernames = {} + self.result_hashed_passwords = {} + + return super().setup() + + def handle_event(self, event): + + if event.type == "EMAIL_ADDRESS": + url = f"{self.base_url}?query=email:{event.data}" + result = self.helpers.request(url, auth=self.auth, headers=self.headers) + if result: + result_json = result.json() + entries = result_json.get("entries") + for i in entries: + user = i.get("username") + pw = i.get("password") + h_pw = i.get("hashed_password") + db_name = i.get("database_name") + + # If there is a password result then adds it to dict + if pw: + self.emit_event(pw, "PASSWORD", source=event) + if pw not in self.result_passwords: + self.result_passwords[pw] = [db_name] + else: + self.result_passwords[pw].append(db_name) + + # If there is a hashed password result then adds it to dict + if h_pw: + if h_pw not in self.result_passwords: + self.result_hashed_passwords[h_pw] = [db_name] + else: + self.result_hashed_passwords[h_pw].append(db_name) + + # If there is a username result then adds it to dict + if user: + if user not in self.result_usernames: + self.result_usernames[user] = [db_name] + else: + self.result_usernames[user].append(db_name) +""" + if self.result_passwords: + for x in self.result_passwords: + self.emit_event(x, "PASSWORD", source=event) + if self.result_hashed_passwords: + #self.hugesuccess(self.result_hashed_passwords) + for x in self.result_hashed_passwords: + self.emit_event(x, "HASHED_PASSWORD", source=event) + if self.result_usernames: + self.hugesuccess(self.result_usernames) +""" \ No newline at end of file From 1ee1caffa35c73221bbbc9d65759f3da5ad41a9a Mon Sep 17 00:00:00 2001 From: SpamFaux Date: Thu, 28 Sep 2023 18:05:05 -0500 Subject: [PATCH 06/49] Created a credshed and dehashed module for bbot. --- bbot/modules/credshed.py | 103 +++++++++++++++++++++++ bbot/modules/dehashed.py | 177 +++++++++++++++++++++++++++++---------- 2 files changed, 235 insertions(+), 45 deletions(-) create mode 100644 bbot/modules/credshed.py diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py new file mode 100644 index 000000000..5b9673b61 --- /dev/null +++ b/bbot/modules/credshed.py @@ -0,0 +1,103 @@ +from bbot.modules.base import BaseModule + +class credshed(BaseModule): + watched_events = ["EMAIL_ADDRESS","DNS_NAME"] + produced_events = ["PASSWORD","HASHED_PASSWORD","USERNAME"] + flags = ["passive"] + meta = {"description":"Send queries to your own credshed server to check for known credentials of your targets", "auth_required": True} + options = {"username":"", "password":"", "credshed_url":""} + + + async def setup(self): + self.base_url = self.config.get("credshed_url","") + self.username = self.config.get("username","") + self.password = self.config.get("password","") + self.results = {} + + auth_setup = await self.helpers.request(f"{self.base_url}/api/auth", + method="POST", + json={ + "username":self.username, + "password":self.password + } + ) + self.auth_token = auth_setup.json().get('access_token') + + return await super().setup() + + async def filter_event(self, event): + if event.module == self or "subdomain" in event.tags: + return False + return True + + async def handle_event(self, event): + cs_query = await self.helpers.request(f"{self.base_url}/api/search", method="POST", cookies={'access_token_cookie':self.auth_token}, json={'query':event.data}) + + if cs_query and cs_query.json().get('stats').get("total_count") > 0: + accounts = cs_query.json().get('accounts') + for i in accounts: + email = i.get("e") + pw = i.get("p") + h_pw = i.get("h") + user = i.get("u") + src = i.get("s")[0] + if email not in self.results: + self.results[email] = { + "source":[src], + "passwords":{}, + "hashed":{}, + "usernames":{} + } + else: + if src not in self.results[email]["source"]: + self.results[email]["source"].append(src) + + if pw: + if pw not in self.results[email]["passwords"]: + self.results[email]["passwords"][pw] = [src] + else: + self.results[email]["passwords"][pw].append(src) + + if h_pw: + for x in h_pw: + if x not in self.results[email]["hashed"]: + self.results[email]["hashed"][x] = [src] + else: + self.results[email]["hashed"][x].append(src) + + if user: + if user not in self.results[email]["usernames"]: + self.results[email]["usernames"][user] = [src] + else: + self.results[email]["usernames"][user].append(src) + + + for x in self.results: + if cs_query.json().get('stats').get('query_type') == "domain": + f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags='credshed') + self.emit_event(f) + + if self.results[x]["hashed"]: + for y in self.results[x]["hashed"]: + self.emit_event(y, "HASHED_PASSWORD",source=f, tags=f'credshed-source-{self.results[x]["hashed"][y]}') + + if self.results[x]["passwords"]: + for y in self.results[x]["passwords"]: + self.emit_event(y, "PASSWORD", source=f, tags=f'credshed-source-{self.results[x]["passwords"][y]}') + + if self.results[x]["usernames"]: + for y in self.results[x]["usernames"]: + self.emit_event(y, "USERNAME", source=f, tags=f'credshed-source-{self.results[x]["usernames"][y]}') + + if cs_query.json().get('stats').get('query_type') == "email": + if self.results[x]["hashed"]: + for y in self.results[x]["hashed"]: + self.emit_event(y, "HASHED_PASSWORD",source=event, tags=f'credshed-source-{self.results[x]["hashed"][y]}') + + if self.results[x]["passwords"]: + for y in self.results[x]["passwords"]: + self.emit_event(y, "PASSWORD", source=event, tags=f'credshed-source-{self.results[x]["passwords"][y]}') + + if self.results[x]["usernames"]: + for y in self.results[x]["usernames"]: + self.emit_event(y, "USERNAME", source=event, tags=f'credshed-source-{self.results[x]["usernames"][y]}') \ No newline at end of file diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index a93029d01..149c59291 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -2,72 +2,159 @@ from requests.auth import HTTPBasicAuth class dehashed(BaseModule): - watched_events = ["EMAIL_ADDRESS"] - produced_events = ["PASSWORD"] + watched_events = ["EMAIL_ADDRESS","DNS_NAME"] + produced_events = ["PASSWORD","HASHED_PASSWORD","USERNAME"] flags = ["passive"] - meta = {"description": "Execute queries against dehash.com for exposed credentials", "auth_required": True} - options = {"username":"", "api_key": ""} - options_desc = {"username":"Email Address associated with your API key","api_key": "API Key"} + meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} + options = {"username":"", "api_key":"", "check_domains":False} + options_desc = {"username":"Email Address associated with your API key","api_key": "API Key", "check_domains":"Enable only if bbot should search dehashed for new email addresses - this can be significantly more expensive against the API."} base_url = "https://api.dehashed.com/search" - def setup(self): + async def setup(self): self.username = self.config.get("username", "") self.api_key = self.config.get("api_key","") self.auth = HTTPBasicAuth(self.username, self.api_key) self.headers = { "Accept": "application/json", } + self.check_domains = self.config.get("check_domains",False) self.result_passwords = {} self.result_usernames = {} self.result_hashed_passwords = {} + self.result_email = {} + self.api_balance = 0 + self.api_count = 0 + self.api_warn = False + + return await super().setup() + + async def filter_event(self, event): + if event.module == self or "subdomain" in event.tags: + return False + return True + + def parse_results(self, result, isDNS): + # If there are no results then leave the function + if result.json().get('total') == 0: + return + + self.api_balance = result.json().get('balance') + self.api_count = result.json().get('total') + + entries = result.json().get("entries") + + + # parses the entries + for i in entries: + user = i.get("username") + pw = i.get("password") + h_pw = i.get("hashed_password") + email = i.get("email") + db_name = i.get("database_name") + + if isDNS: + if email not in self.result_email: + self.result_email[email] = { + "source":[db_name], + "passwords":{}, + "hashed":{}, + "usernames":{} + } + else: + self.result_email[email]["source"].append(db_name) + + if pw: + if pw not in self.result_email[email]["passwords"]: + self.result_email[email]["passwords"][pw] = [db_name] + else: + self.result_email[email]["passwords"][pw].append(db_name) + + if h_pw: + if h_pw not in self.result_email[email]["hashed"]: + self.result_email[email]["hashed"][h_pw] = [db_name] + else: + self.result_email[email]["hashed"][h_pw].append(db_name) + + else: + if pw: + if pw not in self.result_passwords: + self.result_passwords[pw] = [db_name] + else: + self.result_passwords[pw].append(db_name) + + # If there is a hashed password result then adds it to dict + if h_pw: + if h_pw not in self.result_passwords: + self.result_hashed_passwords[h_pw] = [db_name] + else: + self.result_hashed_passwords[h_pw].append(db_name) + + # If there is a username result then adds it to dict + if user: + if user not in self.result_usernames: + self.result_usernames[user] = [db_name] + else: + self.result_usernames[user].append(db_name) + + + + async def handle_event(self, event): + # Checks if user explicitly opted in for domain checking + if self.check_domains: + if event.type == "DNS_NAME": + url = f"{self.base_url}?query=domain:{event.data}&size=10000&page=1" + result = await self.helpers.request(url, auth=self.auth, headers=self.headers) + + if result: + self.parse_results(result,True) + + # Check to see if multiple requests are required + if self.api_count > 10000: + pages = self.api_count // 10000 + if self.api_count % 10000: + pages += 1 + + for i in range(2, pages+1): + if i <= 3: + url = f"{self.base_url}?query=domain:{event.data}&size=10000&page={i}" + result = await self.helpers.request(url, auth=self.auth, headers=self.headers) + self.parse_results(result,True) + # Due to a limitation in the api we cannot request more thank 30k results or 3 pages of 10k - if we go beyond the 3 pages then break out and warn + elif i > 3: + self.api_warn = True + break + + for x in self.result_email: + f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags=self.result_email[x]["source"]) + self.emit_event(f) + if self.result_email[x]["hashed"]: + for y in self.result_email[x]["hashed"]: + self.emit_event(y, "HASHED_PASSWORD", source=f, tags=self.result_email[x]["hashed"][y]) + if self.result_email[x]["passwords"]: + for y in self.result_email[x]["passwords"]: + self.emit_event(y, "PASSWORD", source=f, tags=self.result_email[x]["passwords"][y]) + if self.result_email[x]["usernames"]: + for y in self.result_email[x]["usernames"]: + self.emit_event(y, "USERNAME", source=f, tags=self.result_email[x]["usernames"][y]) - return super().setup() - def handle_event(self, event): - if event.type == "EMAIL_ADDRESS": url = f"{self.base_url}?query=email:{event.data}" - result = self.helpers.request(url, auth=self.auth, headers=self.headers) + result = await self.helpers.request(url, auth=self.auth, headers=self.headers) if result: - result_json = result.json() - entries = result_json.get("entries") - for i in entries: - user = i.get("username") - pw = i.get("password") - h_pw = i.get("hashed_password") - db_name = i.get("database_name") - - # If there is a password result then adds it to dict - if pw: - self.emit_event(pw, "PASSWORD", source=event) - if pw not in self.result_passwords: - self.result_passwords[pw] = [db_name] - else: - self.result_passwords[pw].append(db_name) - - # If there is a hashed password result then adds it to dict - if h_pw: - if h_pw not in self.result_passwords: - self.result_hashed_passwords[h_pw] = [db_name] - else: - self.result_hashed_passwords[h_pw].append(db_name) - - # If there is a username result then adds it to dict - if user: - if user not in self.result_usernames: - self.result_usernames[user] = [db_name] - else: - self.result_usernames[user].append(db_name) -""" + self.parse_results(result,False) + if self.result_passwords: for x in self.result_passwords: - self.emit_event(x, "PASSWORD", source=event) + self.emit_event(x, "PASSWORD", source=event, tags=self.result_passwords[x]) if self.result_hashed_passwords: - #self.hugesuccess(self.result_hashed_passwords) for x in self.result_hashed_passwords: - self.emit_event(x, "HASHED_PASSWORD", source=event) + self.emit_event(x, "HASHED_PASSWORD", source=event, tags=self.result_hashed_passwords[x]) if self.result_usernames: - self.hugesuccess(self.result_usernames) -""" \ No newline at end of file + for x in self.result_usernames: + self.emit_event(x, "USERNAME", source=event, tags=self.result_usernames[x]) + + if self.api_warn: + self.info(f"{event.data} has {self.api_count} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results.") From 0d893577047976c2fd24f9a2f6760a7813bf49bc Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 12:26:13 -0400 Subject: [PATCH 07/49] resolved conflicts --- bbot/core/event/base.py | 5 +- bbot/modules/credshed.py | 140 +++++++++++++++++++++++---------------- bbot/modules/dehashed.py | 60 ++++++++--------- 3 files changed, 117 insertions(+), 88 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 1b4587872..043414ab6 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1128,15 +1128,19 @@ def _pretty_string(self): class GEOLOCATION(BaseEvent): _always_emit = True + class PASSWORD(BaseEvent): _always_emit = True + class HASHED_PASSWORD(BaseEvent): _always_emit = True + class USERNAME(BaseEvent): _always_emit = True + class SOCIAL(DictEvent): _always_emit = True @@ -1163,7 +1167,6 @@ class _data_validator(BaseModel): def _pretty_string(self): return self.data["WAF"] - def make_event( data, event_type=None, diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index 5b9673b61..764468bf1 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -1,40 +1,59 @@ +from contextlib import suppress + from bbot.modules.base import BaseModule -class credshed(BaseModule): - watched_events = ["EMAIL_ADDRESS","DNS_NAME"] - produced_events = ["PASSWORD","HASHED_PASSWORD","USERNAME"] - flags = ["passive"] - meta = {"description":"Send queries to your own credshed server to check for known credentials of your targets", "auth_required": True} - options = {"username":"", "password":"", "credshed_url":""} +class credshed(BaseModule): + watched_events = ["EMAIL_ADDRESS", "DNS_NAME"] + produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME", "EMAIL_ADDRESS"] + flags = ["passive", "safe"] + meta = { + "description": "Send queries to your own credshed server to check for known credentials of your targets", + "auth_required": True, + } + options = {"username": "", "password": "", "credshed_url": ""} + options_desc = { + "username": "Credshed username", + "password": "Credshed password", + "credshed_url": "URL of credshed server", + } async def setup(self): - self.base_url = self.config.get("credshed_url","") - self.username = self.config.get("username","") - self.password = self.config.get("password","") + self.base_url = self.config.get("credshed_url", "").rstrip("/") + self.username = self.config.get("username", "") + self.password = self.config.get("password", "") self.results = {} - auth_setup = await self.helpers.request(f"{self.base_url}/api/auth", - method="POST", - json={ - "username":self.username, - "password":self.password - } - ) - self.auth_token = auth_setup.json().get('access_token') + if not (self.base_url and self.username and self.password): + return None, "Must set username, password, and credshed_url" + + auth_setup = await self.helpers.request( + f"{self.base_url}/api/auth", method="POST", json={"username": self.username, "password": self.password} + ) + self.auth_token = "" + with suppress(Exception): + self.auth_token = auth_setup.json().get("access_token", "") + if not self.auth_token: + return None, f"Failed to retrieve credshed auth token from url: {self.base_url}" return await super().setup() - + async def filter_event(self, event): if event.module == self or "subdomain" in event.tags: return False return True - + async def handle_event(self, event): - cs_query = await self.helpers.request(f"{self.base_url}/api/search", method="POST", cookies={'access_token_cookie':self.auth_token}, json={'query':event.data}) - - if cs_query and cs_query.json().get('stats').get("total_count") > 0: - accounts = cs_query.json().get('accounts') + self.critical(event) + cs_query = await self.helpers.request( + f"{self.base_url}/api/search", + method="POST", + cookies={"access_token_cookie": self.auth_token}, + json={"query": event.data}, + ) + + if cs_query and cs_query.json().get("stats").get("total_count") > 0: + accounts = cs_query.json().get("accounts") for i in accounts: email = i.get("e") pw = i.get("p") @@ -42,12 +61,7 @@ async def handle_event(self, event): user = i.get("u") src = i.get("s")[0] if email not in self.results: - self.results[email] = { - "source":[src], - "passwords":{}, - "hashed":{}, - "usernames":{} - } + self.results[email] = {"source": [src], "passwords": {}, "hashed": {}, "usernames": {}} else: if src not in self.results[email]["source"]: self.results[email]["source"].append(src) @@ -57,47 +71,61 @@ async def handle_event(self, event): self.results[email]["passwords"][pw] = [src] else: self.results[email]["passwords"][pw].append(src) - + if h_pw: for x in h_pw: if x not in self.results[email]["hashed"]: self.results[email]["hashed"][x] = [src] else: self.results[email]["hashed"][x].append(src) - + if user: if user not in self.results[email]["usernames"]: self.results[email]["usernames"][user] = [src] else: self.results[email]["usernames"][user].append(src) - for x in self.results: - if cs_query.json().get('stats').get('query_type') == "domain": - f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags='credshed') - self.emit_event(f) - - if self.results[x]["hashed"]: - for y in self.results[x]["hashed"]: - self.emit_event(y, "HASHED_PASSWORD",source=f, tags=f'credshed-source-{self.results[x]["hashed"][y]}') - - if self.results[x]["passwords"]: - for y in self.results[x]["passwords"]: - self.emit_event(y, "PASSWORD", source=f, tags=f'credshed-source-{self.results[x]["passwords"][y]}') - - if self.results[x]["usernames"]: - for y in self.results[x]["usernames"]: - self.emit_event(y, "USERNAME", source=f, tags=f'credshed-source-{self.results[x]["usernames"][y]}') - - if cs_query.json().get('stats').get('query_type') == "email": + if cs_query.json().get("stats").get("query_type") == "domain": + f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags="credshed") + self.emit_event(f) + if self.results[x]["hashed"]: - for y in self.results[x]["hashed"]: - self.emit_event(y, "HASHED_PASSWORD",source=event, tags=f'credshed-source-{self.results[x]["hashed"][y]}') + for y in self.results[x]["hashed"]: + self.emit_event( + y, "HASHED_PASSWORD", source=f, tags=f'credshed-source-{self.results[x]["hashed"][y]}' + ) if self.results[x]["passwords"]: - for y in self.results[x]["passwords"]: - self.emit_event(y, "PASSWORD", source=event, tags=f'credshed-source-{self.results[x]["passwords"][y]}') - + for y in self.results[x]["passwords"]: + self.emit_event( + y, "PASSWORD", source=f, tags=f'credshed-source-{self.results[x]["passwords"][y]}' + ) + + if self.results[x]["usernames"]: + for y in self.results[x]["usernames"]: + self.emit_event( + y, "USERNAME", source=f, tags=f'credshed-source-{self.results[x]["usernames"][y]}' + ) + + if cs_query.json().get("stats").get("query_type") == "email": + if self.results[x]["hashed"]: + for y in self.results[x]["hashed"]: + self.emit_event( + y, + "HASHED_PASSWORD", + source=event, + tags=f'credshed-source-{self.results[x]["hashed"][y]}', + ) + + if self.results[x]["passwords"]: + for y in self.results[x]["passwords"]: + self.emit_event( + y, "PASSWORD", source=event, tags=f'credshed-source-{self.results[x]["passwords"][y]}' + ) + if self.results[x]["usernames"]: - for y in self.results[x]["usernames"]: - self.emit_event(y, "USERNAME", source=event, tags=f'credshed-source-{self.results[x]["usernames"][y]}') \ No newline at end of file + for y in self.results[x]["usernames"]: + self.emit_event( + y, "USERNAME", source=event, tags=f'credshed-source-{self.results[x]["usernames"][y]}' + ) diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 149c59291..2dabcc988 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -1,24 +1,29 @@ from bbot.modules.base import BaseModule from requests.auth import HTTPBasicAuth + class dehashed(BaseModule): - watched_events = ["EMAIL_ADDRESS","DNS_NAME"] - produced_events = ["PASSWORD","HASHED_PASSWORD","USERNAME"] + watched_events = ["EMAIL_ADDRESS", "DNS_NAME"] + produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"] flags = ["passive"] meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} - options = {"username":"", "api_key":"", "check_domains":False} - options_desc = {"username":"Email Address associated with your API key","api_key": "API Key", "check_domains":"Enable only if bbot should search dehashed for new email addresses - this can be significantly more expensive against the API."} + options = {"username": "", "api_key": "", "check_domains": False} + options_desc = { + "username": "Email Address associated with your API key", + "api_key": "API Key", + "check_domains": "Enable only if bbot should search dehashed for new email addresses - this can be significantly more expensive against the API.", + } base_url = "https://api.dehashed.com/search" async def setup(self): self.username = self.config.get("username", "") - self.api_key = self.config.get("api_key","") + self.api_key = self.config.get("api_key", "") self.auth = HTTPBasicAuth(self.username, self.api_key) self.headers = { "Accept": "application/json", } - self.check_domains = self.config.get("check_domains",False) + self.check_domains = self.config.get("check_domains", False) self.result_passwords = {} self.result_usernames = {} @@ -37,14 +42,13 @@ async def filter_event(self, event): def parse_results(self, result, isDNS): # If there are no results then leave the function - if result.json().get('total') == 0: + if result.json().get("total") == 0: return - - self.api_balance = result.json().get('balance') - self.api_count = result.json().get('total') + + self.api_balance = result.json().get("balance") + self.api_count = result.json().get("total") entries = result.json().get("entries") - # parses the entries for i in entries: @@ -56,12 +60,7 @@ def parse_results(self, result, isDNS): if isDNS: if email not in self.result_email: - self.result_email[email] = { - "source":[db_name], - "passwords":{}, - "hashed":{}, - "usernames":{} - } + self.result_email[email] = {"source": [db_name], "passwords": {}, "hashed": {}, "usernames": {}} else: self.result_email[email]["source"].append(db_name) @@ -70,27 +69,27 @@ def parse_results(self, result, isDNS): self.result_email[email]["passwords"][pw] = [db_name] else: self.result_email[email]["passwords"][pw].append(db_name) - + if h_pw: if h_pw not in self.result_email[email]["hashed"]: self.result_email[email]["hashed"][h_pw] = [db_name] else: self.result_email[email]["hashed"][h_pw].append(db_name) - else: + else: if pw: if pw not in self.result_passwords: self.result_passwords[pw] = [db_name] else: self.result_passwords[pw].append(db_name) - + # If there is a hashed password result then adds it to dict if h_pw: if h_pw not in self.result_passwords: self.result_hashed_passwords[h_pw] = [db_name] else: self.result_hashed_passwords[h_pw].append(db_name) - + # If there is a username result then adds it to dict if user: if user not in self.result_usernames: @@ -98,17 +97,15 @@ def parse_results(self, result, isDNS): else: self.result_usernames[user].append(db_name) - - async def handle_event(self, event): # Checks if user explicitly opted in for domain checking if self.check_domains: if event.type == "DNS_NAME": url = f"{self.base_url}?query=domain:{event.data}&size=10000&page=1" result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - + if result: - self.parse_results(result,True) + self.parse_results(result, True) # Check to see if multiple requests are required if self.api_count > 10000: @@ -116,11 +113,11 @@ async def handle_event(self, event): if self.api_count % 10000: pages += 1 - for i in range(2, pages+1): + for i in range(2, pages + 1): if i <= 3: url = f"{self.base_url}?query=domain:{event.data}&size=10000&page={i}" result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - self.parse_results(result,True) + self.parse_results(result, True) # Due to a limitation in the api we cannot request more thank 30k results or 3 pages of 10k - if we go beyond the 3 pages then break out and warn elif i > 3: self.api_warn = True @@ -139,12 +136,11 @@ async def handle_event(self, event): for y in self.result_email[x]["usernames"]: self.emit_event(y, "USERNAME", source=f, tags=self.result_email[x]["usernames"][y]) - if event.type == "EMAIL_ADDRESS": url = f"{self.base_url}?query=email:{event.data}" result = await self.helpers.request(url, auth=self.auth, headers=self.headers) if result: - self.parse_results(result,False) + self.parse_results(result, False) if self.result_passwords: for x in self.result_passwords: @@ -155,6 +151,8 @@ async def handle_event(self, event): if self.result_usernames: for x in self.result_usernames: self.emit_event(x, "USERNAME", source=event, tags=self.result_usernames[x]) - + if self.api_warn: - self.info(f"{event.data} has {self.api_count} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results.") + self.info( + f"{event.data} has {self.api_count} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results." + ) From 90002e90faa37e56fa32e906252e9fa2f2c7ffae Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 11 Oct 2023 16:21:41 -0400 Subject: [PATCH 08/49] removed debugging statement --- bbot/modules/credshed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index 764468bf1..d25b3f3b4 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -44,7 +44,6 @@ async def filter_event(self, event): return True async def handle_event(self, event): - self.critical(event) cs_query = await self.helpers.request( f"{self.base_url}/api/search", method="POST", From 5eaf82ed123263e5c7a0d13c86b424da0aeb174d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 11 Oct 2023 16:35:25 -0400 Subject: [PATCH 09/49] removed debugging statement --- bbot/modules/dehashed.py | 82 ++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 2dabcc988..9c2a3be27 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -10,8 +10,8 @@ class dehashed(BaseModule): options = {"username": "", "api_key": "", "check_domains": False} options_desc = { "username": "Email Address associated with your API key", - "api_key": "API Key", - "check_domains": "Enable only if bbot should search dehashed for new email addresses - this can be significantly more expensive against the API.", + "api_key": "DeHashed API Key", + "check_domains": "Enable only if BBOT should search dehashed for new email addresses - this can be significantly more expensive against the API.", } base_url = "https://api.dehashed.com/search" @@ -98,45 +98,45 @@ def parse_results(self, result, isDNS): self.result_usernames[user].append(db_name) async def handle_event(self, event): - # Checks if user explicitly opted in for domain checking - if self.check_domains: - if event.type == "DNS_NAME": - url = f"{self.base_url}?query=domain:{event.data}&size=10000&page=1" - result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - - if result: - self.parse_results(result, True) - - # Check to see if multiple requests are required - if self.api_count > 10000: - pages = self.api_count // 10000 - if self.api_count % 10000: - pages += 1 - - for i in range(2, pages + 1): - if i <= 3: - url = f"{self.base_url}?query=domain:{event.data}&size=10000&page={i}" - result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - self.parse_results(result, True) - # Due to a limitation in the api we cannot request more thank 30k results or 3 pages of 10k - if we go beyond the 3 pages then break out and warn - elif i > 3: - self.api_warn = True - break - - for x in self.result_email: - f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags=self.result_email[x]["source"]) - self.emit_event(f) - if self.result_email[x]["hashed"]: - for y in self.result_email[x]["hashed"]: - self.emit_event(y, "HASHED_PASSWORD", source=f, tags=self.result_email[x]["hashed"][y]) - if self.result_email[x]["passwords"]: - for y in self.result_email[x]["passwords"]: - self.emit_event(y, "PASSWORD", source=f, tags=self.result_email[x]["passwords"][y]) - if self.result_email[x]["usernames"]: - for y in self.result_email[x]["usernames"]: - self.emit_event(y, "USERNAME", source=f, tags=self.result_email[x]["usernames"][y]) - - if event.type == "EMAIL_ADDRESS": + if event.type == "DNS_NAME": + url = f"{self.base_url}?query=domain:{event.data}&size=10000&page=1" + result = await self.helpers.request(url, auth=self.auth, headers=self.headers) + self.hugesuccess(result.json()) + return + + if result: + self.parse_results(result, True) + + # Check to see if multiple requests are required + if self.api_count > 10000: + pages = self.api_count // 10000 + if self.api_count % 10000: + pages += 1 + + for i in range(2, pages + 1): + if i <= 3: + url = f"{self.base_url}?query=domain:{event.data}&size=10000&page={i}" + result = await self.helpers.request(url, auth=self.auth, headers=self.headers) + self.parse_results(result, True) + # Due to a limitation in the api we cannot request more thank 30k results or 3 pages of 10k - if we go beyond the 3 pages then break out and warn + elif i > 3: + self.api_warn = True + break + + for x in self.result_email: + f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags=self.result_email[x]["source"]) + self.emit_event(f) + if self.result_email[x]["hashed"]: + for y in self.result_email[x]["hashed"]: + self.emit_event(y, "HASHED_PASSWORD", source=f, tags=self.result_email[x]["hashed"][y]) + if self.result_email[x]["passwords"]: + for y in self.result_email[x]["passwords"]: + self.emit_event(y, "PASSWORD", source=f, tags=self.result_email[x]["passwords"][y]) + if self.result_email[x]["usernames"]: + for y in self.result_email[x]["usernames"]: + self.emit_event(y, "USERNAME", source=f, tags=self.result_email[x]["usernames"][y]) + + elif event.type == "EMAIL_ADDRESS": url = f"{self.base_url}?query=email:{event.data}" result = await self.helpers.request(url, auth=self.auth, headers=self.headers) if result: From 82748d97c73b91f6aeb0d82f6ed9594bc00b9a5a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 12:31:28 -0400 Subject: [PATCH 10/49] resolved conflicts --- bbot/core/helpers/regexes.py | 2 +- bbot/modules/credshed.py | 4 +- bbot/modules/dehashed.py | 221 ++++++++++++++--------------------- 3 files changed, 94 insertions(+), 133 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index f4a122348..313626094 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -31,7 +31,7 @@ _hostname_regex = r"(?!\w*\.\w+)\w(?:[\w-]{0,100}\w)?" hostname_regex = re.compile(r"^" + _hostname_regex + r"$", re.I) -_email_regex = r"(?:[^\W_][\w\-\.\+]{,100})@" + _dns_name_regex +_email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex email_regex = re.compile(_email_regex, re.I) _ptr_regex = r"(?:[0-9]{1,3}[-_\.]){3}[0-9]{1,3}" diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index d25b3f3b4..7eb704881 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -24,6 +24,7 @@ async def setup(self): self.password = self.config.get("password", "") self.results = {} + # make sure we have the information required to make queries if not (self.base_url and self.username and self.password): return None, "Must set username, password, and credshed_url" @@ -33,8 +34,9 @@ async def setup(self): self.auth_token = "" with suppress(Exception): self.auth_token = auth_setup.json().get("access_token", "") + # hard-fail if we didn't get an access token if not self.auth_token: - return None, f"Failed to retrieve credshed auth token from url: {self.base_url}" + return False, f"Failed to retrieve credshed auth token from url: {self.base_url}" return await super().setup() diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 9c2a3be27..5d4fbdc20 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -1,17 +1,17 @@ +from contextlib import suppress + from bbot.modules.base import BaseModule -from requests.auth import HTTPBasicAuth class dehashed(BaseModule): - watched_events = ["EMAIL_ADDRESS", "DNS_NAME"] + watched_events = ["DNS_NAME"] produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"] flags = ["passive"] meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} - options = {"username": "", "api_key": "", "check_domains": False} + options = {"username": "", "api_key": ""} options_desc = { "username": "Email Address associated with your API key", - "api_key": "DeHashed API Key", - "check_domains": "Enable only if BBOT should search dehashed for new email addresses - this can be significantly more expensive against the API.", + "api_key": "DeHashed API Key" } base_url = "https://api.dehashed.com/search" @@ -19,140 +19,99 @@ class dehashed(BaseModule): async def setup(self): self.username = self.config.get("username", "") self.api_key = self.config.get("api_key", "") - self.auth = HTTPBasicAuth(self.username, self.api_key) + self.auth = (self.username, self.api_key) self.headers = { "Accept": "application/json", } - self.check_domains = self.config.get("check_domains", False) + self.queries_processed = set() + self.data_seen = set() - self.result_passwords = {} - self.result_usernames = {} - self.result_hashed_passwords = {} - self.result_email = {} - self.api_balance = 0 - self.api_count = 0 - self.api_warn = False + # soft-fail if we don't have the necessary information to make queries + if not (self.username and self.api_key): + return None, "No username / API key set" return await super().setup() async def filter_event(self, event): - if event.module == self or "subdomain" in event.tags: - return False - return True - - def parse_results(self, result, isDNS): - # If there are no results then leave the function - if result.json().get("total") == 0: - return - - self.api_balance = result.json().get("balance") - self.api_count = result.json().get("total") - - entries = result.json().get("entries") - - # parses the entries - for i in entries: - user = i.get("username") - pw = i.get("password") - h_pw = i.get("hashed_password") - email = i.get("email") - db_name = i.get("database_name") - - if isDNS: - if email not in self.result_email: - self.result_email[email] = {"source": [db_name], "passwords": {}, "hashed": {}, "usernames": {}} - else: - self.result_email[email]["source"].append(db_name) - - if pw: - if pw not in self.result_email[email]["passwords"]: - self.result_email[email]["passwords"][pw] = [db_name] - else: - self.result_email[email]["passwords"][pw].append(db_name) - - if h_pw: - if h_pw not in self.result_email[email]["hashed"]: - self.result_email[email]["hashed"][h_pw] = [db_name] - else: - self.result_email[email]["hashed"][h_pw].append(db_name) - - else: - if pw: - if pw not in self.result_passwords: - self.result_passwords[pw] = [db_name] - else: - self.result_passwords[pw].append(db_name) - - # If there is a hashed password result then adds it to dict - if h_pw: - if h_pw not in self.result_passwords: - self.result_hashed_passwords[h_pw] = [db_name] - else: - self.result_hashed_passwords[h_pw].append(db_name) - - # If there is a username result then adds it to dict - if user: - if user not in self.result_usernames: - self.result_usernames[user] = [db_name] - else: - self.result_usernames[user].append(db_name) + query = self.make_query(event) + query_hash = hash(query) + if query_hash not in self.queries_processed: + self.queries_processed.add(query_hash) + return True + return False, f'Already processed "{query}"' async def handle_event(self, event): + already_seen = set() + emails = {} + if event.type == "DNS_NAME": - url = f"{self.base_url}?query=domain:{event.data}&size=10000&page=1" - result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - self.hugesuccess(result.json()) - return - - if result: - self.parse_results(result, True) - - # Check to see if multiple requests are required - if self.api_count > 10000: - pages = self.api_count // 10000 - if self.api_count % 10000: - pages += 1 - - for i in range(2, pages + 1): - if i <= 3: - url = f"{self.base_url}?query=domain:{event.data}&size=10000&page={i}" - result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - self.parse_results(result, True) - # Due to a limitation in the api we cannot request more thank 30k results or 3 pages of 10k - if we go beyond the 3 pages then break out and warn - elif i > 3: - self.api_warn = True - break - - for x in self.result_email: - f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags=self.result_email[x]["source"]) - self.emit_event(f) - if self.result_email[x]["hashed"]: - for y in self.result_email[x]["hashed"]: - self.emit_event(y, "HASHED_PASSWORD", source=f, tags=self.result_email[x]["hashed"][y]) - if self.result_email[x]["passwords"]: - for y in self.result_email[x]["passwords"]: - self.emit_event(y, "PASSWORD", source=f, tags=self.result_email[x]["passwords"][y]) - if self.result_email[x]["usernames"]: - for y in self.result_email[x]["usernames"]: - self.emit_event(y, "USERNAME", source=f, tags=self.result_email[x]["usernames"][y]) - - elif event.type == "EMAIL_ADDRESS": - url = f"{self.base_url}?query=email:{event.data}" - result = await self.helpers.request(url, auth=self.auth, headers=self.headers) - if result: - self.parse_results(result, False) - - if self.result_passwords: - for x in self.result_passwords: - self.emit_event(x, "PASSWORD", source=event, tags=self.result_passwords[x]) - if self.result_hashed_passwords: - for x in self.result_hashed_passwords: - self.emit_event(x, "HASHED_PASSWORD", source=event, tags=self.result_hashed_passwords[x]) - if self.result_usernames: - for x in self.result_usernames: - self.emit_event(x, "USERNAME", source=event, tags=self.result_usernames[x]) - - if self.api_warn: - self.info( - f"{event.data} has {self.api_count} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results." - ) + query = f"domain:{event.data}" + else: + query = f"email:{event.data}" + url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}" + async for entries in self.query(url): + for entry in entries: + + # we have to clean up the email field because dehashed does a poor job of it + email_str = entry.get("email", "").replace('\\', '') + found_emails = list(self.helpers.extract_emails(email_str)) + if not found_emails: + self.debug(f"Invalid email from dehashed.com: {email_str}") + continue + email = found_emails[0] + + user = entry.get("username", "") + pw = entry.get("password", "") + h_pw = entry.get("hashed_password", "") + db_name = entry.get("database_name", "") + + tags = [] + if db_name: + tags = [f"db-{db_name}"] + if email: + email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags) + if email_event is not None: + self.emit_event(email_event) + if user and not self.already_seen(f"{email}:{user}"): + self.emit_event(user, "USERNAME", source=email_event, tags=tags) + if pw and not self.already_seen(f"{email}:{pw}"): + self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) + if h_pw and not self.already_seen(f"{email}:{h_pw}"): + self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) + + def already_seen(self, item): + h = hash(item) + already_seen = h in self.data_seen + self.data_seen.add(h) + return already_seen + + async def query(self, url): + page = 0 + num_entries = 0 + agen = self.helpers.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False) + async for result in agen: + result_json = {} + with suppress(Exception): + result_json = result.json() + total = result_json.get("total", 0) + entries = result_json.get("entries", []) + if entries is None: + entries = [] + num_entries += len(entries) + page += 1 + if (page >= 3) or (not entries): + if result is not None and result.status_code != 200: + self.warning(f"Error retrieving results from dehashed.com: {result.text}") + elif (page >= 3) and (total > num_entries): + self.info( + f"{event.data} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results." + ) + agen.aclose() + break + yield entries + + def make_query(self, event): + if "target" in event.tags: + return event.data + _, domain = self.helpers.split_domain(event.data) + return domain From 3d316cab2a023677c79f4231e5162ae4e23ff204 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 12:24:37 -0400 Subject: [PATCH 11/49] dehashed module tests --- .../module_tests/test_module_dehashed.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 bbot/test/test_step_2/module_tests/test_module_dehashed.py diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py new file mode 100644 index 000000000..767884bd5 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py @@ -0,0 +1,61 @@ +from .base import ModuleTestBase + +dehashed_domain_response = { + "balance": 10000, + "entries": [ + { + "id": "4363462346", + "email": "bob@blacklanternsecurity.com", + "ip_address": "", + "username": "", + "password": "", + "hashed_password": "$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve", + "name": "Bob Smith", + "vin": "", + "address": "", + "phone": "+91283423839", + "database_name": "eatstreet", + }, + { + "id": "234623453454", + "email": "tim@blacklanternsecurity.com", + "ip_address": "", + "username": "timmy", + "password": "TimTamSlam69", + "hashed_password": "", + "name": "Tim Tam", + "vin": "", + "address": "", + "phone": "+123455667", + "database_name": "eatstreet", + }, + ], + "success": True, + "took": "61µs", + "total": 2, +} + + +class TestDehashed(ModuleTestBase): + config_overrides = {"modules": {"dehashed": {"username": "admin", "api_key": "deadbeef"}}} + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://api.dehashed.com/search?query=domain:blacklanternsecurity.com&size=10000&page=1", + json=dehashed_domain_response, + ) + + def check(self, module_test, events): + assert len(events) == 7 + assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"]) + assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"]) + assert 1 == len( + [ + e + for e in events + if e.type == "HASHED_PASSWORD" + and e.data == "$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve" + ] + ) + assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"]) + assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "timmy"]) From d4568bbfe807f7942aa155a9267312cd2a382ddd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 14:27:35 -0400 Subject: [PATCH 12/49] credshed module tests, optimizations --- bbot/core/event/base.py | 1 + bbot/modules/credshed.py | 126 ++++++------------ bbot/modules/dehashed.py | 44 ++---- bbot/modules/templates/credential_leak.py | 33 +++++ .../module_tests/test_module_credshed.py | 91 +++++++++++++ 5 files changed, 172 insertions(+), 123 deletions(-) create mode 100644 bbot/modules/templates/credential_leak.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_credshed.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 043414ab6..c24601fef 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1167,6 +1167,7 @@ class _data_validator(BaseModel): def _pretty_string(self): return self.data["WAF"] + def make_event( data, event_type=None, diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index 7eb704881..468e3180f 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -1,10 +1,10 @@ from contextlib import suppress -from bbot.modules.base import BaseModule +from bbot.modules.templates.credential_leak import credential_leak -class credshed(BaseModule): - watched_events = ["EMAIL_ADDRESS", "DNS_NAME"] +class credshed(credential_leak): + watched_events = ["DNS_NAME"] produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME", "EMAIL_ADDRESS"] flags = ["passive", "safe"] meta = { @@ -22,9 +22,8 @@ async def setup(self): self.base_url = self.config.get("credshed_url", "").rstrip("/") self.username = self.config.get("username", "") self.password = self.config.get("password", "") - self.results = {} - # make sure we have the information required to make queries + # soft-fail if we don't have the necessary information to make queries if not (self.base_url and self.username and self.password): return None, "Must set username, password, and credshed_url" @@ -40,93 +39,46 @@ async def setup(self): return await super().setup() - async def filter_event(self, event): - if event.module == self or "subdomain" in event.tags: - return False - return True - async def handle_event(self, event): + query = self.make_query(event) cs_query = await self.helpers.request( f"{self.base_url}/api/search", method="POST", cookies={"access_token_cookie": self.auth_token}, - json={"query": event.data}, + json={"query": query}, ) - if cs_query and cs_query.json().get("stats").get("total_count") > 0: - accounts = cs_query.json().get("accounts") - for i in accounts: - email = i.get("e") - pw = i.get("p") - h_pw = i.get("h") - user = i.get("u") - src = i.get("s")[0] - if email not in self.results: - self.results[email] = {"source": [src], "passwords": {}, "hashed": {}, "usernames": {}} - else: - if src not in self.results[email]["source"]: - self.results[email]["source"].append(src) - - if pw: - if pw not in self.results[email]["passwords"]: - self.results[email]["passwords"][pw] = [src] - else: - self.results[email]["passwords"][pw].append(src) - - if h_pw: - for x in h_pw: - if x not in self.results[email]["hashed"]: - self.results[email]["hashed"][x] = [src] - else: - self.results[email]["hashed"][x].append(src) - - if user: - if user not in self.results[email]["usernames"]: - self.results[email]["usernames"][user] = [src] - else: - self.results[email]["usernames"][user].append(src) - - for x in self.results: - if cs_query.json().get("stats").get("query_type") == "domain": - f = self.make_event(x, "EMAIL_ADDRESS", source=event, tags="credshed") - self.emit_event(f) - - if self.results[x]["hashed"]: - for y in self.results[x]["hashed"]: - self.emit_event( - y, "HASHED_PASSWORD", source=f, tags=f'credshed-source-{self.results[x]["hashed"][y]}' - ) + if cs_query is not None and cs_query.status_code != 200: + self.warning(f"Error retrieving results from {self.base_url} (status code {cs_query.status_code}): {cs_query.text}") - if self.results[x]["passwords"]: - for y in self.results[x]["passwords"]: - self.emit_event( - y, "PASSWORD", source=f, tags=f'credshed-source-{self.results[x]["passwords"][y]}' - ) - - if self.results[x]["usernames"]: - for y in self.results[x]["usernames"]: - self.emit_event( - y, "USERNAME", source=f, tags=f'credshed-source-{self.results[x]["usernames"][y]}' - ) - - if cs_query.json().get("stats").get("query_type") == "email": - if self.results[x]["hashed"]: - for y in self.results[x]["hashed"]: - self.emit_event( - y, - "HASHED_PASSWORD", - source=event, - tags=f'credshed-source-{self.results[x]["hashed"][y]}', - ) - - if self.results[x]["passwords"]: - for y in self.results[x]["passwords"]: - self.emit_event( - y, "PASSWORD", source=event, tags=f'credshed-source-{self.results[x]["passwords"][y]}' - ) - - if self.results[x]["usernames"]: - for y in self.results[x]["usernames"]: - self.emit_event( - y, "USERNAME", source=event, tags=f'credshed-source-{self.results[x]["usernames"][y]}' - ) + json_result = {} + with suppress(Exception): + json_result = cs_query.json() + + if not json_result: + return + + accounts = json_result.get("accounts", []) + + for i in accounts: + email = i.get("e", "") + pw = i.get("p", "") + hashes = i.get("h", []) + user = i.get("u", "") + src = i.get("s", []) + src = [src[0] if src else ""] + + tags = [] + if src: + tags = [f"credshed-source-{src}"] + + email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags) + if email_event is not None: + self.emit_event(email_event) + if user and not self.already_seen(f"{email}:{user}"): + self.emit_event(user, "USERNAME", source=email_event, tags=tags) + if pw and not self.already_seen(f"{email}:{pw}"): + self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) + for h_pw in hashes: + if h_pw and not self.already_seen(f"{email}:{h_pw}"): + self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 5d4fbdc20..06f2ac376 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -1,18 +1,15 @@ from contextlib import suppress -from bbot.modules.base import BaseModule +from bbot.modules.templates.credential_leak import credential_leak -class dehashed(BaseModule): +class dehashed(credential_leak): watched_events = ["DNS_NAME"] produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"] flags = ["passive"] meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} options = {"username": "", "api_key": ""} - options_desc = { - "username": "Email Address associated with your API key", - "api_key": "DeHashed API Key" - } + options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"} base_url = "https://api.dehashed.com/search" @@ -23,8 +20,6 @@ async def setup(self): self.headers = { "Accept": "application/json", } - self.queries_processed = set() - self.data_seen = set() # soft-fail if we don't have the necessary information to make queries if not (self.username and self.api_key): @@ -32,28 +27,15 @@ async def setup(self): return await super().setup() - async def filter_event(self, event): - query = self.make_query(event) - query_hash = hash(query) - if query_hash not in self.queries_processed: - self.queries_processed.add(query_hash) - return True - return False, f'Already processed "{query}"' - async def handle_event(self, event): already_seen = set() emails = {} - - if event.type == "DNS_NAME": - query = f"domain:{event.data}" - else: - query = f"email:{event.data}" + query = f"domain:{self.make_query(event)}" url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}" async for entries in self.query(url): for entry in entries: - # we have to clean up the email field because dehashed does a poor job of it - email_str = entry.get("email", "").replace('\\', '') + email_str = entry.get("email", "").replace("\\", "") found_emails = list(self.helpers.extract_emails(email_str)) if not found_emails: self.debug(f"Invalid email from dehashed.com: {email_str}") @@ -79,12 +61,6 @@ async def handle_event(self, event): if h_pw and not self.already_seen(f"{email}:{h_pw}"): self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) - def already_seen(self, item): - h = hash(item) - already_seen = h in self.data_seen - self.data_seen.add(h) - return already_seen - async def query(self, url): page = 0 num_entries = 0 @@ -101,7 +77,9 @@ async def query(self, url): page += 1 if (page >= 3) or (not entries): if result is not None and result.status_code != 200: - self.warning(f"Error retrieving results from dehashed.com: {result.text}") + self.warning( + f"Error retrieving results from dehashed.com (status code {results.status_code}): {result.text}" + ) elif (page >= 3) and (total > num_entries): self.info( f"{event.data} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results." @@ -109,9 +87,3 @@ async def query(self, url): agen.aclose() break yield entries - - def make_query(self, event): - if "target" in event.tags: - return event.data - _, domain = self.helpers.split_domain(event.data) - return domain diff --git a/bbot/modules/templates/credential_leak.py b/bbot/modules/templates/credential_leak.py new file mode 100644 index 000000000..5085e197a --- /dev/null +++ b/bbot/modules/templates/credential_leak.py @@ -0,0 +1,33 @@ +from bbot.modules.base import BaseModule + + +class credential_leak(BaseModule): + """ + A typical free API-based subdomain enumeration module + Inherited by many other modules including sublist3r, dnsdumpster, etc. + """ + + async def setup(self): + self.queries_processed = set() + self.data_seen = set() + return True + + async def filter_event(self, event): + query = self.make_query(event) + query_hash = hash(query) + if query_hash not in self.queries_processed: + self.queries_processed.add(query_hash) + return True + return False, f'Already processed "{query}"' + + def make_query(self, event): + if "target" in event.tags: + return event.data + _, domain = self.helpers.split_domain(event.data) + return domain + + def already_seen(self, item): + h = hash(item) + already_seen = h in self.data_seen + self.data_seen.add(h) + return already_seen diff --git a/bbot/test/test_step_2/module_tests/test_module_credshed.py b/bbot/test/test_step_2/module_tests/test_module_credshed.py new file mode 100644 index 000000000..7de642412 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_credshed.py @@ -0,0 +1,91 @@ +from .base import ModuleTestBase + + +credshed_auth_response = { + "access_token": "big_access_token", + "login": True, +} + + +credshed_response = { + "accounts": [ + { + "e": "bob@blacklanternsecurity.com", + "h": [], + "m": "hello my name is bob", + "p": "", + "s": [121562], + "u": "", + }, + { + "e": "judy@blacklanternsecurity.com", + "h": [ + "539FE8942DEADBEEFBC49E6EB2F175AC", + "D2D8F0E9A4A2DEADBEEF1AC80F36D61F", + "$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm", + ], + "m": "hello my name is judy", + "p": "", + "s": [80437], + "u": "", + }, + { + "e": "tim@blacklanternsecurity.com", + "h": [], + "m": "hello my name is tim", + "p": "TimTamSlam69", + "s": [80437], + "u": "tim", + }, + ], + "stats": { + "accounts_searched": 9820758365, + "elapsed": "0.00", + "limit": 1000, + "query": "blacklanternsecurity.com", + "query_type": "domain", + "sources_searched": 129957, + "total_count": 3, + "unique_count": 3, + }, +} + + +class TestCredshed(ModuleTestBase): + config_overrides = { + "modules": {"credshed": {"username": "admin", "password": "password", "credshed_url": "https://credshed.com"}} + } + + async def setup_before_prep(self, module_test): + module_test.httpx_mock.add_response( + url=f"https://credshed.com/api/auth", + json=credshed_auth_response, + method="POST", + ) + module_test.httpx_mock.add_response( + url=f"https://credshed.com/api/search", + json=credshed_response, + method="POST", + ) + + def check(self, module_test, events): + assert len(events) == 10 + assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"]) + assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "judy@blacklanternsecurity.com"]) + assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"]) + assert 1 == len( + [e for e in events if e.type == "HASHED_PASSWORD" and e.data == "539FE8942DEADBEEFBC49E6EB2F175AC"] + ) + assert 1 == len( + [e for e in events if e.type == "HASHED_PASSWORD" and e.data == "D2D8F0E9A4A2DEADBEEF1AC80F36D61F"] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "HASHED_PASSWORD" + and e.data == "$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm" + ] + ) + assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"]) + assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "tim"]) From ee948671155b9c750f194de383d2b2501b26f39d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 14:30:56 -0400 Subject: [PATCH 13/49] flake8 --- bbot/modules/dehashed.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 06f2ac376..cdfa70741 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -9,7 +9,10 @@ class dehashed(credential_leak): flags = ["passive"] meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} options = {"username": "", "api_key": ""} - options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"} + options_desc = { + "username": "Email Address associated with your API key", + "api_key": "DeHashed API Key" + } base_url = "https://api.dehashed.com/search" @@ -28,14 +31,10 @@ async def setup(self): return await super().setup() async def handle_event(self, event): - already_seen = set() - emails = {} - query = f"domain:{self.make_query(event)}" - url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}" - async for entries in self.query(url): + async for entries in self.query(event): for entry in entries: # we have to clean up the email field because dehashed does a poor job of it - email_str = entry.get("email", "").replace("\\", "") + email_str = entry.get("email", "").replace('\\', '') found_emails = list(self.helpers.extract_emails(email_str)) if not found_emails: self.debug(f"Invalid email from dehashed.com: {email_str}") @@ -61,7 +60,9 @@ async def handle_event(self, event): if h_pw and not self.already_seen(f"{email}:{h_pw}"): self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) - async def query(self, url): + async def query(self, event): + query = f"domain:{self.make_query(event)}" + url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}" page = 0 num_entries = 0 agen = self.helpers.api_page_iter(url=url, auth=self.auth, headers=self.headers, json=False) @@ -77,9 +78,7 @@ async def query(self, url): page += 1 if (page >= 3) or (not entries): if result is not None and result.status_code != 200: - self.warning( - f"Error retrieving results from dehashed.com (status code {results.status_code}): {result.text}" - ) + self.warning(f"Error retrieving results from dehashed.com (status code {result.status_code}): {result.text}") elif (page >= 3) and (total > num_entries): self.info( f"{event.data} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results." From 880051510099b2dc2f7c63fa41496def5ed0be73 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 14:39:52 -0400 Subject: [PATCH 14/49] update README --- README.md | 106 ++++++++++++++++++++++++++++++++++++++ docs/scanning/advanced.md | 14 ++--- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8818a53a8..167629059 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,112 @@ asyncio.run(main()) - [Troubleshooting](https://www.blacklanternsecurity.com/bbot/troubleshooting) +## List of BBOT Modules + +| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | +|----------------------|----------|-----------------|------------------------------------------------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------| +| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | +| bucket_aws | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_gcp | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | +| dnszonetransfer | scan | No | Attempt DNS zone transfers | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | +| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | +| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | +| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | +| git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | +| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | +| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | +| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | +| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | +| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic, web-thorough | URL | URL_HINT | +| masscan | scan | No | Port scan IP subnets with masscan | active, aggressive, portscan | SCAN | OPEN_TCP_PORT | +| nmap | scan | No | Execute port scans with nmap | active, aggressive, portscan, web-thorough | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT | +| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic, web-thorough | HTTP_RESPONSE, URL | DNS_NAME, FINDING | +| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, VULNERABILITY | +| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | +| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic, web-thorough | URL | URL_UNVERIFIED | +| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | +| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | +| social | scan | No | Look for social media links in webpages | active, safe, social-enum | URL_UNVERIFIED | SOCIAL | +| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | +| subdomain_hijack | scan | No | Detect hijackable subdomains | active, cloud-enum, safe, subdomain-enum, subdomain-hijack, web-basic, web-thorough | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING | +| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, slow, web-thorough | URL | FINDING, VULNERABILITY | +| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | +| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | +| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | +| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic, web-thorough | HTTP_RESPONSE | TECHNOLOGY | +| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | +| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | +| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | +| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| crobat | scan | No | Query Project Crobat for subdomains | passive, safe | DNS_NAME | DNS_NAME | +| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dnscommonsrv | scan | No | Check for common SRV records | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| github | scan | Yes | Query Github's API for related repositories | passive, safe, subdomain-enum | DNS_NAME | URL_UNVERIFIED | +| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | +| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | +| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | +| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | +| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| massdns | scan | No | Brute-force subdomains with massdns (highly effective) | aggressive, passive, slow, subdomain-enum | DNS_NAME | DNS_NAME | +| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| nsec | scan | No | Enumerate subdomains by NSEC-walking | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| riddler | scan | No | Query riddler.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| sublist3r | scan | No | Query sublist3r's API for subdomains | passive, safe | DNS_NAME | DNS_NAME | +| threatminer | scan | No | Query threatminer's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | +| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| asset_inventory | output | No | Output to an asset inventory style flattened CSV file | | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY | IP_ADDRESS, OPEN_TCP_PORT | +| csv | output | No | Output to CSV | | * | | +| discord | output | No | Message a Discord channel when certain events are encountered | | * | | +| http | output | No | Send every event to a custom URL via a web request | | * | | +| human | output | No | Output to text | | * | | +| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | +| neo4j | output | No | Output to Neo4j | | * | | +| python | output | No | Output via Python API | | * | | +| slack | output | No | Message a Slack channel when certain events are encountered | | * | | +| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | +| teams | output | No | Message a Slack channel when certain events are encountered | | * | | +| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | +| websocket | output | No | Output to websockets | | * | | +| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | +| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED | +| speculate | internal | No | Derive certain event types from others by common sense | passive | DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, STORAGE_BUCKET, URL, URL_UNVERIFIED | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT | + + ## Acknowledgements Thanks to these amazing people for contributing to BBOT! :heart: diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 2509946bd..31f4275b5 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -33,16 +33,10 @@ asyncio.run(main()) ```text -usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] - [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] - [--strict-scope] [-m MODULE [MODULE ...]] [-l] - [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] - [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] - [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] - [-o DIR] [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] - [--dry-run] [--current-config] - [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] - [-a] [--version] +usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-m MODULE [MODULE ...]] [-l] + [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] [-o DIR] + [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-config] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] + [--version] Bighuge BLS OSINT Tool From d9c28a32937315b2a222cc094e0429f2271954b2 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 14:41:06 -0400 Subject: [PATCH 15/49] move module list to bottom of readme --- README.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 167629059..f97b22955 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,26 @@ asyncio.run(main()) - [Troubleshooting](https://www.blacklanternsecurity.com/bbot/troubleshooting) +## Acknowledgements + +Thanks to these amazing people for contributing to BBOT! :heart: + +If you're interested in contributing to BBOT, or just curious how it works under the hood, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). + +

+ + + +

+ +Special thanks to the following people who made BBOT possible: + +- @TheTechromancer for creating [BBOT](https://github.com/blacklanternsecurity/bbot) +- @liquidsec for his extensive work on BBOT's web hacking features, including [badsecrets](https://github.com/blacklanternsecurity/badsecrets) +- Steve Micallef (@smicallef) for creating Spiderfoot +- @kerrymilan for his Neo4j and Ansible expertise +- Aleksei Kornev (@alekseiko) for allowing us ownership of the bbot Pypi repository <3 + ## List of BBOT Modules | Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | @@ -272,23 +292,3 @@ asyncio.run(main()) | excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED | | speculate | internal | No | Derive certain event types from others by common sense | passive | DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, STORAGE_BUCKET, URL, URL_UNVERIFIED | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT | - -## Acknowledgements - -Thanks to these amazing people for contributing to BBOT! :heart: - -If you're interested in contributing to BBOT, or just curious how it works under the hood, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). - -

- - - -

- -Special thanks to the following people who made BBOT possible: - -- @TheTechromancer for creating [BBOT](https://github.com/blacklanternsecurity/bbot) -- @liquidsec for his extensive work on BBOT's web hacking features, including [badsecrets](https://github.com/blacklanternsecurity/badsecrets) -- Steve Micallef (@smicallef) for creating Spiderfoot -- @kerrymilan for his Neo4j and Ansible expertise -- Aleksei Kornev (@alekseiko) for allowing us ownership of the bbot Pypi repository <3 From a08f85481cb98544772cbbaa39cb0f1f32e80c0d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 12 Oct 2023 14:42:18 -0400 Subject: [PATCH 16/49] blacked --- bbot/modules/credshed.py | 4 +++- bbot/modules/dehashed.py | 11 +++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index 468e3180f..c493199d7 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -49,7 +49,9 @@ async def handle_event(self, event): ) if cs_query is not None and cs_query.status_code != 200: - self.warning(f"Error retrieving results from {self.base_url} (status code {cs_query.status_code}): {cs_query.text}") + self.warning( + f"Error retrieving results from {self.base_url} (status code {cs_query.status_code}): {cs_query.text}" + ) json_result = {} with suppress(Exception): diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index cdfa70741..113652490 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -9,10 +9,7 @@ class dehashed(credential_leak): flags = ["passive"] meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} options = {"username": "", "api_key": ""} - options_desc = { - "username": "Email Address associated with your API key", - "api_key": "DeHashed API Key" - } + options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"} base_url = "https://api.dehashed.com/search" @@ -34,7 +31,7 @@ async def handle_event(self, event): async for entries in self.query(event): for entry in entries: # we have to clean up the email field because dehashed does a poor job of it - email_str = entry.get("email", "").replace('\\', '') + email_str = entry.get("email", "").replace("\\", "") found_emails = list(self.helpers.extract_emails(email_str)) if not found_emails: self.debug(f"Invalid email from dehashed.com: {email_str}") @@ -78,7 +75,9 @@ async def query(self, event): page += 1 if (page >= 3) or (not entries): if result is not None and result.status_code != 200: - self.warning(f"Error retrieving results from dehashed.com (status code {result.status_code}): {result.text}") + self.warning( + f"Error retrieving results from dehashed.com (status code {result.status_code}): {result.text}" + ) elif (page >= 3) and (total > num_entries): self.info( f"{event.data} has {total:,} results in Dehashed. The API can only process the first 30,000 results. Please check dehashed.com to get the remaining results." From 0c006ddc9ba8fc3568c78dad78e4b5762c3f263b Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Fri, 13 Oct 2023 14:30:10 +0000 Subject: [PATCH 17/49] Refresh module docs --- docs/modules/list_of_modules.md | 1 + docs/scanning/advanced.md | 2 +- docs/scanning/configuration.md | 296 ++++++++++++++++---------------- docs/scanning/events.md | 4 +- docs/scanning/index.md | 44 ++--- 5 files changed, 175 insertions(+), 172 deletions(-) diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 4b9a21a21..9a4e23da5 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -13,6 +13,7 @@ | dnszonetransfer | scan | No | Attempt DNS zone transfers | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | | ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | | ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | +| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, safe | HTTP_RESPONSE, URL_UNVERIFIED | | | fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | | generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | | git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 2509946bd..1da57f619 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -61,7 +61,7 @@ Target: Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] - Modules to enable. Choices: affiliates,anubisdb,asn,azure_realm,azure_tenant,badsecrets,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_digitalocean,bucket_firebase,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,crobat,crt,digitorus,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,fingerprintx,fullhunt,generic_ssrf,git,github,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,nmap,nsec,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomain_hijack,subdomaincenter,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye + Modules to enable. Choices: affiliates,anubisdb,asn,azure_realm,azure_tenant,badsecrets,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_digitalocean,bucket_firebase,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,crobat,crt,digitorus,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,github,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,nmap,nsec,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomain_hijack,subdomaincenter,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye -l, --list-modules List available modules. -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index c85796860..25403fc02 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -203,151 +203,153 @@ dns_omit_queries: Many modules accept their own configuration options. These options have the ability to change their behavior. For example, the `nmap` module accepts options for `ports`, `timing`, etc. Below is a list of all possible module config options. -| Config Option | Type | Description | Default | -|------------------------------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| modules.bucket_aws.permutations | bool | Whether to try permutations | False | -| modules.bucket_azure.permutations | bool | Whether to try permutations | False | -| modules.bucket_digitalocean.permutations | bool | Whether to try permutations | False | -| modules.bucket_firebase.permutations | bool | Whether to try permutations | False | -| modules.bucket_gcp.permutations | bool | Whether to try permutations | False | -| modules.dnszonetransfer.timeout | int | Max seconds to wait before timing out | 10 | -| modules.ffuf.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | | -| modules.ffuf.lines | int | take only the first N lines from the wordlist when finding directories | 5000 | -| modules.ffuf.max_depth | int | the maxium directory depth to attempt to solve | 0 | -| modules.ffuf.version | str | ffuf version | 2.0.0 | -| modules.ffuf.wordlist | str | Specify wordlist to use when finding directories | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt | -| modules.ffuf_shortnames.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | | -| modules.ffuf_shortnames.find_common_prefixes | bool | Attempt to automatically detect common prefixes and make additional ffuf runs against them | False | -| modules.ffuf_shortnames.find_delimeters | bool | Attempt to detect common delimeters and make additional ffuf runs against them | True | -| modules.ffuf_shortnames.ignore_redirects | bool | Explicitly ignore redirects (301,302) | True | -| modules.ffuf_shortnames.lines | int | take only the first N lines from the wordlist when finding directories | 1000000 | -| modules.ffuf_shortnames.max_depth | int | the maxium directory depth to attempt to solve | 1 | -| modules.ffuf_shortnames.version | str | ffuf version | 2.0.0 | -| modules.ffuf_shortnames.wordlist | str | Specify wordlist to use when finding directories | | -| modules.ffuf_shortnames.wordlist_extensions | str | Specify wordlist to use when making extension lists | | -| modules.fingerprintx.version | str | fingerprintx version | 1.1.4 | -| modules.gowitness.output_path | str | where to save screenshots | | -| modules.gowitness.resolution_x | int | screenshot resolution x | 1440 | -| modules.gowitness.resolution_y | int | screenshot resolution y | 900 | -| modules.gowitness.threads | int | threads used to run | 4 | -| modules.gowitness.timeout | int | preflight check timeout | 10 | -| modules.gowitness.version | str | gowitness version | 2.4.2 | -| modules.httpx.in_scope_only | bool | Only visit web resources that are in scope. | True | -| modules.httpx.max_response_size | int | Max response size in bytes | 5242880 | -| modules.httpx.threads | int | Number of httpx threads to use | 50 | -| modules.httpx.version | str | httpx version | 1.2.5 | -| modules.iis_shortnames.detect_only | bool | Only detect the vulnerability and do not run the shortname scanner | True | -| modules.iis_shortnames.max_node_count | int | Limit how many nodes to attempt to resolve on any given recursion branch | 30 | -| modules.masscan.ping_first | bool | Only portscan hosts that reply to pings | False | -| modules.masscan.ports | str | Ports to scan | 80,443 | -| modules.masscan.rate | int | Rate in packets per second | 600 | -| modules.masscan.use_cache | bool | Instead of scanning, use the results from the previous scan | False | -| modules.masscan.wait | int | Seconds to wait for replies after scan is complete | 10 | -| modules.nmap.ports | str | ports to scan | | -| modules.nmap.skip_host_discovery | bool | skip host discovery (-Pn) | True | -| modules.nmap.timing | str |` -T<0-5>: Set timing template (higher is faster) `| T4 | -| modules.nmap.top_ports | int | top ports to scan | 100 | -| modules.ntlm.try_all | bool | Try every NTLM endpoint | False | -| modules.nuclei.budget | int | Used in budget mode to set the number of requests which will be alloted to the nuclei scan | 1 | -| modules.nuclei.concurrency | int | maximum number of templates to be executed in parallel (default 25) | 25 | -| modules.nuclei.directory_only | bool | Filter out 'file' URL event (default True) | True | -| modules.nuclei.etags | str | tags to exclude from the scan | | -| modules.nuclei.mode | str | manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual | -| modules.nuclei.ratelimit | int | maximum number of requests to send per second (default 150) | 150 | -| modules.nuclei.severity | str | Filter based on severity field available in the template. | | -| modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | -| modules.nuclei.templates | str | template or template directory paths to include in the scan | | -| modules.nuclei.version | str | nuclei version | 2.9.15 | -| modules.oauth.try_all | bool | Check for OAUTH/IODC on every subdomain and URL. | False | -| modules.paramminer_cookies.http_extract | bool | Attempt to find additional wordlist words from the HTTP Response | True | -| modules.paramminer_cookies.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | -| modules.paramminer_cookies.wordlist | str | Define the wordlist to be used to derive cookies | | -| modules.paramminer_getparams.http_extract | bool | Attempt to find additional wordlist words from the HTTP Response | True | -| modules.paramminer_getparams.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | -| modules.paramminer_getparams.wordlist | str | Define the wordlist to be used to derive headers | | -| modules.paramminer_headers.http_extract | bool | Attempt to find additional wordlist words from the HTTP Response | True | -| modules.paramminer_headers.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | -| modules.paramminer_headers.wordlist | str | Define the wordlist to be used to derive headers | | -| modules.robots.include_allow | bool | Include 'Allow' Entries | True | -| modules.robots.include_disallow | bool | Include 'Disallow' Entries | True | -| modules.robots.include_sitemap | bool | Include 'sitemap' entries | False | -| modules.secretsdb.min_confidence | int | Only use signatures with this confidence score or higher | 99 | -| modules.secretsdb.signatures | str | File path or URL to YAML signatures | https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml | -| modules.sslcert.skip_non_ssl | bool | Don't try common non-SSL ports | True | -| modules.sslcert.timeout | float | Socket connect timeout in seconds | 5.0 | -| modules.subdomain_hijack.fingerprints | str | URL or path to fingerprints.json | https://raw.githubusercontent.com/EdOverflow/can-i-take-over-xyz/master/fingerprints.json | -| modules.telerik.exploit_RAU_crypto | bool | Attempt to confirm any RAU AXD detections are vulnerable | False | -| modules.url_manipulation.allow_redirects | bool | Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default. | True | -| modules.vhost.force_basehost | str | Use a custom base host (e.g. evilcorp.com) instead of the default behavior of using the current URL | | -| modules.vhost.lines | int | take only the first N lines from the wordlist when finding directories | 5000 | -| modules.vhost.wordlist | str | Wordlist containing subdomains | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | -| modules.wafw00f.generic_detect | bool | When no specific WAF detections are made, try to peform a generic detect | True | -| modules.bevigil.api_key | str | BeVigil OSINT API Key | | -| modules.bevigil.urls | bool | Emit URLs in addition to DNS_NAMEs | False | -| modules.binaryedge.api_key | str | BinaryEdge API key | | -| modules.binaryedge.max_records | int | Limit results to help prevent exceeding API quota | 1000 | -| modules.builtwith.api_key | str | Builtwith API key | | -| modules.builtwith.redirects | bool | Also look up inbound and outbound redirects | True | -| modules.c99.api_key | str | c99.nl API key | | -| modules.censys.api_id | str | Censys.io API ID | | -| modules.censys.api_secret | str | Censys.io API Secret | | -| modules.censys.max_pages | int | Maximum number of pages to fetch (100 results per page) | 5 | -| modules.chaos.api_key | str | Chaos API key | | -| modules.fullhunt.api_key | str | FullHunt API Key | | -| modules.github.api_key | str | Github token | | -| modules.hunterio.api_key | str | Hunter.IO API key | | -| modules.ip2location.api_key | str | IP2location.io API Key | | -| modules.ip2location.lang | str | Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name. | | -| modules.ipneighbor.num_bits | int | Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts) | 4 | -| modules.ipstack.api_key | str | IPStack GeoIP API Key | | -| modules.leakix.api_key | str | LeakIX API Key | | -| modules.massdns.max_mutations | int | Max number of smart mutations per subdomain | 500 | -| modules.massdns.max_resolvers | int | Number of concurrent massdns resolvers | 1000 | -| modules.massdns.wordlist | str | Subdomain wordlist URL | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | -| modules.passivetotal.api_key | str | RiskIQ API Key | | -| modules.passivetotal.username | str | RiskIQ Username | | -| modules.pgp.search_urls | list | PGP key servers to search |` ['https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=', 'http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search='] `| -| modules.securitytrails.api_key | str | SecurityTrails API key | | -| modules.shodan_dns.api_key | str | Shodan API key | | -| modules.urlscan.urls | bool | Emit URLs in addition to DNS_NAMEs | False | -| modules.virustotal.api_key | str | VirusTotal API Key | | -| modules.wayback.garbage_threshold | int | Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data) | 10 | -| modules.wayback.urls | bool | emit URLs in addition to DNS_NAMEs | False | -| modules.zoomeye.api_key | str | ZoomEye API key | | -| modules.zoomeye.include_related | bool | Include domains which may be related to the target | False | -| modules.zoomeye.max_pages | int | How many pages of results to fetch | 20 | -| output_modules.asset_inventory.output_file | str | Set a custom output file | | -| output_modules.asset_inventory.summary_netmask | int | Subnet mask to use when summarizing IP addresses at end of scan | 16 | -| output_modules.asset_inventory.use_previous | bool |` Emit previous asset inventory as new events (use in conjunction with -n ) `| False | -| output_modules.csv.output_file | str | Output to CSV file | | -| output_modules.discord.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | -| output_modules.discord.min_severity | str | Only allow VULNERABILITY events of this severity or highter | LOW | -| output_modules.discord.webhook_url | str | Discord webhook URL | | -| output_modules.http.bearer | str | Authorization Bearer token | | -| output_modules.http.method | str | HTTP method | POST | -| output_modules.http.password | str | Password (basic auth) | | -| output_modules.http.timeout | int | HTTP timeout | 10 | -| output_modules.http.url | str | Web URL | | -| output_modules.http.username | str | Username (basic auth) | | -| output_modules.human.console | bool | Output to console | True | -| output_modules.human.output_file | str | Output to file | | -| output_modules.json.console | bool | Output to console | False | -| output_modules.json.output_file | str | Output to file | | -| output_modules.neo4j.password | str | Neo4j password | bbotislife | -| output_modules.neo4j.uri | str | Neo4j server + port | bolt://localhost:7687 | -| output_modules.neo4j.username | str | Neo4j username | neo4j | -| output_modules.slack.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | -| output_modules.slack.min_severity | str | Only allow VULNERABILITY events of this severity or highter | LOW | -| output_modules.slack.webhook_url | str | Discord webhook URL | | -| output_modules.subdomains.include_unresolved | bool | Include unresolved subdomains in output | False | -| output_modules.subdomains.output_file | str | Output to file | | -| output_modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | -| output_modules.teams.min_severity | str | Only allow VULNERABILITY events of this severity or highter | LOW | -| output_modules.teams.webhook_url | str | Discord webhook URL | | -| output_modules.web_report.css_theme_file | str | CSS theme URL for HTML output | https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css | -| output_modules.web_report.output_file | str | Output to file | | -| output_modules.websocket.token | str | Authorization Bearer token | | -| output_modules.websocket.url | str | Web URL | | -| internal_modules.speculate.max_hosts | int | Max number of IP_RANGE hosts to convert into IP_ADDRESS events | 65536 | -| internal_modules.speculate.ports | str | The set of ports to speculate on | 80,443 | +| Config Option | Type | Description | Default | +|------------------------------------------------|--------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| modules.bucket_aws.permutations | bool | Whether to try permutations | False | +| modules.bucket_azure.permutations | bool | Whether to try permutations | False | +| modules.bucket_digitalocean.permutations | bool | Whether to try permutations | False | +| modules.bucket_firebase.permutations | bool | Whether to try permutations | False | +| modules.bucket_gcp.permutations | bool | Whether to try permutations | False | +| modules.dnszonetransfer.timeout | int | Max seconds to wait before timing out | 10 | +| modules.ffuf.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | | +| modules.ffuf.lines | int | take only the first N lines from the wordlist when finding directories | 5000 | +| modules.ffuf.max_depth | int | the maxium directory depth to attempt to solve | 0 | +| modules.ffuf.version | str | ffuf version | 2.0.0 | +| modules.ffuf.wordlist | str | Specify wordlist to use when finding directories | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt | +| modules.ffuf_shortnames.extensions | str | Optionally include a list of extensions to extend the keyword with (comma separated) | | +| modules.ffuf_shortnames.find_common_prefixes | bool | Attempt to automatically detect common prefixes and make additional ffuf runs against them | False | +| modules.ffuf_shortnames.find_delimeters | bool | Attempt to detect common delimeters and make additional ffuf runs against them | True | +| modules.ffuf_shortnames.ignore_redirects | bool | Explicitly ignore redirects (301,302) | True | +| modules.ffuf_shortnames.lines | int | take only the first N lines from the wordlist when finding directories | 1000000 | +| modules.ffuf_shortnames.max_depth | int | the maxium directory depth to attempt to solve | 1 | +| modules.ffuf_shortnames.version | str | ffuf version | 2.0.0 | +| modules.ffuf_shortnames.wordlist | str | Specify wordlist to use when finding directories | | +| modules.ffuf_shortnames.wordlist_extensions | str | Specify wordlist to use when making extension lists | | +| modules.filedownload.extensions | list | File extensions to download | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'exe', 'ica', 'indd', 'ini', 'jar', 'key', 'pub', 'log', 'markdown', 'md', 'msi', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'png', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'raw', 'rdp', 'sh', 'sql', 'swp', 'sxw', 'tar', 'tar.gz', 'zip', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | +| modules.filedownload.max_filesize | str | Cancel download if filesize is greater than this size | 10MB | +| modules.fingerprintx.version | str | fingerprintx version | 1.1.4 | +| modules.gowitness.output_path | str | where to save screenshots | | +| modules.gowitness.resolution_x | int | screenshot resolution x | 1440 | +| modules.gowitness.resolution_y | int | screenshot resolution y | 900 | +| modules.gowitness.threads | int | threads used to run | 4 | +| modules.gowitness.timeout | int | preflight check timeout | 10 | +| modules.gowitness.version | str | gowitness version | 2.4.2 | +| modules.httpx.in_scope_only | bool | Only visit web resources that are in scope. | True | +| modules.httpx.max_response_size | int | Max response size in bytes | 5242880 | +| modules.httpx.threads | int | Number of httpx threads to use | 50 | +| modules.httpx.version | str | httpx version | 1.2.5 | +| modules.iis_shortnames.detect_only | bool | Only detect the vulnerability and do not run the shortname scanner | True | +| modules.iis_shortnames.max_node_count | int | Limit how many nodes to attempt to resolve on any given recursion branch | 30 | +| modules.masscan.ping_first | bool | Only portscan hosts that reply to pings | False | +| modules.masscan.ports | str | Ports to scan | 80,443 | +| modules.masscan.rate | int | Rate in packets per second | 600 | +| modules.masscan.use_cache | bool | Instead of scanning, use the results from the previous scan | False | +| modules.masscan.wait | int | Seconds to wait for replies after scan is complete | 10 | +| modules.nmap.ports | str | ports to scan | | +| modules.nmap.skip_host_discovery | bool | skip host discovery (-Pn) | True | +| modules.nmap.timing | str |` -T<0-5>: Set timing template (higher is faster) `| T4 | +| modules.nmap.top_ports | int | top ports to scan | 100 | +| modules.ntlm.try_all | bool | Try every NTLM endpoint | False | +| modules.nuclei.budget | int | Used in budget mode to set the number of requests which will be alloted to the nuclei scan | 1 | +| modules.nuclei.concurrency | int | maximum number of templates to be executed in parallel (default 25) | 25 | +| modules.nuclei.directory_only | bool | Filter out 'file' URL event (default True) | True | +| modules.nuclei.etags | str | tags to exclude from the scan | | +| modules.nuclei.mode | str | manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual | +| modules.nuclei.ratelimit | int | maximum number of requests to send per second (default 150) | 150 | +| modules.nuclei.severity | str | Filter based on severity field available in the template. | | +| modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | +| modules.nuclei.templates | str | template or template directory paths to include in the scan | | +| modules.nuclei.version | str | nuclei version | 2.9.15 | +| modules.oauth.try_all | bool | Check for OAUTH/IODC on every subdomain and URL. | False | +| modules.paramminer_cookies.http_extract | bool | Attempt to find additional wordlist words from the HTTP Response | True | +| modules.paramminer_cookies.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | +| modules.paramminer_cookies.wordlist | str | Define the wordlist to be used to derive cookies | | +| modules.paramminer_getparams.http_extract | bool | Attempt to find additional wordlist words from the HTTP Response | True | +| modules.paramminer_getparams.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | +| modules.paramminer_getparams.wordlist | str | Define the wordlist to be used to derive headers | | +| modules.paramminer_headers.http_extract | bool | Attempt to find additional wordlist words from the HTTP Response | True | +| modules.paramminer_headers.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | +| modules.paramminer_headers.wordlist | str | Define the wordlist to be used to derive headers | | +| modules.robots.include_allow | bool | Include 'Allow' Entries | True | +| modules.robots.include_disallow | bool | Include 'Disallow' Entries | True | +| modules.robots.include_sitemap | bool | Include 'sitemap' entries | False | +| modules.secretsdb.min_confidence | int | Only use signatures with this confidence score or higher | 99 | +| modules.secretsdb.signatures | str | File path or URL to YAML signatures | https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml | +| modules.sslcert.skip_non_ssl | bool | Don't try common non-SSL ports | True | +| modules.sslcert.timeout | float | Socket connect timeout in seconds | 5.0 | +| modules.subdomain_hijack.fingerprints | str | URL or path to fingerprints.json | https://raw.githubusercontent.com/EdOverflow/can-i-take-over-xyz/master/fingerprints.json | +| modules.telerik.exploit_RAU_crypto | bool | Attempt to confirm any RAU AXD detections are vulnerable | False | +| modules.url_manipulation.allow_redirects | bool | Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default. | True | +| modules.vhost.force_basehost | str | Use a custom base host (e.g. evilcorp.com) instead of the default behavior of using the current URL | | +| modules.vhost.lines | int | take only the first N lines from the wordlist when finding directories | 5000 | +| modules.vhost.wordlist | str | Wordlist containing subdomains | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | +| modules.wafw00f.generic_detect | bool | When no specific WAF detections are made, try to peform a generic detect | True | +| modules.bevigil.api_key | str | BeVigil OSINT API Key | | +| modules.bevigil.urls | bool | Emit URLs in addition to DNS_NAMEs | False | +| modules.binaryedge.api_key | str | BinaryEdge API key | | +| modules.binaryedge.max_records | int | Limit results to help prevent exceeding API quota | 1000 | +| modules.builtwith.api_key | str | Builtwith API key | | +| modules.builtwith.redirects | bool | Also look up inbound and outbound redirects | True | +| modules.c99.api_key | str | c99.nl API key | | +| modules.censys.api_id | str | Censys.io API ID | | +| modules.censys.api_secret | str | Censys.io API Secret | | +| modules.censys.max_pages | int | Maximum number of pages to fetch (100 results per page) | 5 | +| modules.chaos.api_key | str | Chaos API key | | +| modules.fullhunt.api_key | str | FullHunt API Key | | +| modules.github.api_key | str | Github token | | +| modules.hunterio.api_key | str | Hunter.IO API key | | +| modules.ip2location.api_key | str | IP2location.io API Key | | +| modules.ip2location.lang | str | Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name. | | +| modules.ipneighbor.num_bits | int | Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts) | 4 | +| modules.ipstack.api_key | str | IPStack GeoIP API Key | | +| modules.leakix.api_key | str | LeakIX API Key | | +| modules.massdns.max_mutations | int | Max number of smart mutations per subdomain | 500 | +| modules.massdns.max_resolvers | int | Number of concurrent massdns resolvers | 1000 | +| modules.massdns.wordlist | str | Subdomain wordlist URL | https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt | +| modules.passivetotal.api_key | str | RiskIQ API Key | | +| modules.passivetotal.username | str | RiskIQ Username | | +| modules.pgp.search_urls | list | PGP key servers to search |` ['https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=', 'http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search='] `| +| modules.securitytrails.api_key | str | SecurityTrails API key | | +| modules.shodan_dns.api_key | str | Shodan API key | | +| modules.urlscan.urls | bool | Emit URLs in addition to DNS_NAMEs | False | +| modules.virustotal.api_key | str | VirusTotal API Key | | +| modules.wayback.garbage_threshold | int | Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data) | 10 | +| modules.wayback.urls | bool | emit URLs in addition to DNS_NAMEs | False | +| modules.zoomeye.api_key | str | ZoomEye API key | | +| modules.zoomeye.include_related | bool | Include domains which may be related to the target | False | +| modules.zoomeye.max_pages | int | How many pages of results to fetch | 20 | +| output_modules.asset_inventory.output_file | str | Set a custom output file | | +| output_modules.asset_inventory.summary_netmask | int | Subnet mask to use when summarizing IP addresses at end of scan | 16 | +| output_modules.asset_inventory.use_previous | bool |` Emit previous asset inventory as new events (use in conjunction with -n ) `| False | +| output_modules.csv.output_file | str | Output to CSV file | | +| output_modules.discord.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | +| output_modules.discord.min_severity | str | Only allow VULNERABILITY events of this severity or highter | LOW | +| output_modules.discord.webhook_url | str | Discord webhook URL | | +| output_modules.http.bearer | str | Authorization Bearer token | | +| output_modules.http.method | str | HTTP method | POST | +| output_modules.http.password | str | Password (basic auth) | | +| output_modules.http.timeout | int | HTTP timeout | 10 | +| output_modules.http.url | str | Web URL | | +| output_modules.http.username | str | Username (basic auth) | | +| output_modules.human.console | bool | Output to console | True | +| output_modules.human.output_file | str | Output to file | | +| output_modules.json.console | bool | Output to console | False | +| output_modules.json.output_file | str | Output to file | | +| output_modules.neo4j.password | str | Neo4j password | bbotislife | +| output_modules.neo4j.uri | str | Neo4j server + port | bolt://localhost:7687 | +| output_modules.neo4j.username | str | Neo4j username | neo4j | +| output_modules.slack.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | +| output_modules.slack.min_severity | str | Only allow VULNERABILITY events of this severity or highter | LOW | +| output_modules.slack.webhook_url | str | Discord webhook URL | | +| output_modules.subdomains.include_unresolved | bool | Include unresolved subdomains in output | False | +| output_modules.subdomains.output_file | str | Output to file | | +| output_modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | +| output_modules.teams.min_severity | str | Only allow VULNERABILITY events of this severity or highter | LOW | +| output_modules.teams.webhook_url | str | Discord webhook URL | | +| output_modules.web_report.css_theme_file | str | CSS theme URL for HTML output | https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css | +| output_modules.web_report.output_file | str | Output to file | | +| output_modules.websocket.token | str | Authorization Bearer token | | +| output_modules.websocket.url | str | Web URL | | +| internal_modules.speculate.max_hosts | int | Max number of IP_RANGE hosts to convert into IP_ADDRESS events | 65536 | +| internal_modules.speculate.ports | str | The set of ports to speculate on | 80,443 | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index bf6f5de73..fc05eb822 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -58,7 +58,7 @@ Below is a full list of event types along with which modules produce/consume the | EMAIL_ADDRESS | 0 | 5 | | emailformat, hunterio, pgp, skymem, sslcert | | FINDING | 2 | 21 | asset_inventory, web_report | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, git, host_header, hunt, ntlm, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, smuggler, speculate, subdomain_hijack, telerik, url_manipulation | | GEOLOCATION | 0 | 2 | | ip2location, ipstack | -| HTTP_RESPONSE | 11 | 1 | badsecrets, excavate, host_header, hunt, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, wappalyzer | httpx | +| HTTP_RESPONSE | 12 | 1 | badsecrets, excavate, filedownload, host_header, hunt, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, wappalyzer | httpx | | IP_ADDRESS | 7 | 3 | asn, asset_inventory, ip2location, ipneighbor, ipstack, nmap, speculate | asset_inventory, ipneighbor, speculate | | IP_RANGE | 1 | 0 | speculate | | | OPEN_TCP_PORT | 4 | 4 | asset_inventory, fingerprintx, httpx, sslcert | asset_inventory, masscan, nmap, speculate | @@ -69,7 +69,7 @@ Below is a full list of event types along with which modules produce/consume the | TECHNOLOGY | 2 | 2 | asset_inventory, web_report | gowitness, wappalyzer | | URL | 18 | 2 | asset_inventory, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | | URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | -| URL_UNVERIFIED | 4 | 11 | httpx, oauth, social, speculate | azure_realm, bevigil, excavate, ffuf, ffuf_shortnames, github, gowitness, hunterio, robots, urlscan, wayback | +| URL_UNVERIFIED | 5 | 11 | filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, excavate, ffuf, ffuf_shortnames, github, gowitness, hunterio, robots, urlscan, wayback | | VHOST | 1 | 1 | web_report | vhost | | VULNERABILITY | 2 | 4 | asset_inventory, web_report | badsecrets, generic_ssrf, nuclei, telerik | | WAF | 0 | 1 | | wafw00f | diff --git a/docs/scanning/index.md b/docs/scanning/index.md index f47c50ba9..2be84977f 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -107,28 +107,28 @@ A single module can have multiple flags. For example, the `securitytrails` modul ### List of Flags -| Flag | # Modules | Description | Modules | -|------------------|-------------|-----------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| safe | 67 | Non-intrusive, safe to run | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, badsecrets, bevigil, binaryedge, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, emailformat, fingerprintx, fullhunt, git, github, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, leakix, myssl, nsec, ntlm, oauth, otx, passivetotal, pgp, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomain_hijack, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | -| passive | 50 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, github, hackertarget, hunterio, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, nsec, otx, passivetotal, pgp, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 44 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, github, hackertarget, httpx, hunterio, ipneighbor, leakix, massdns, myssl, nsec, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomain_hijack, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | -| active | 37 | Makes active connections to target systems | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, dnszonetransfer, ffuf, ffuf_shortnames, fingerprintx, generic_ssrf, git, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, social, sslcert, subdomain_hijack, telerik, url_manipulation, vhost, wafw00f, wappalyzer | -| web-thorough | 24 | More advanced web scanning functionality | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, ffuf_shortnames, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, wappalyzer | -| aggressive | 18 | Generates a large amount of network traffic | bypass403, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | -| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, badsecrets, bucket_aws, bucket_azure, bucket_firebase, bucket_gcp, git, httpx, hunt, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, subdomain_hijack, wappalyzer | -| cloud-enum | 10 | Enumerates cloud resources | azure_realm, azure_tenant, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, httpx, oauth, subdomain_hijack | -| slow | 9 | May take a long time to complete | bucket_digitalocean, fingerprintx, massdns, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, vhost | -| affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | -| email-enum | 5 | Enumerates email addresses | emailformat, hunterio, pgp, skymem, sslcert | -| deadly | 3 | Highly aggressive | ffuf, nuclei, vhost | -| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | -| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | -| portscan | 2 | Discovers open ports | masscan, nmap | -| report | 2 | Generates a report at the end of the scan | affiliates, asn | -| social-enum | 2 | Enumerates social media | httpx, social | -| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | -| subdomain-hijack | 1 | Detects hijackable subdomains | subdomain_hijack | -| web-screenshots | 1 | Takes screenshots of web pages | gowitness | +| Flag | # Modules | Description | Modules | +|------------------|-------------|-----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| safe | 68 | Non-intrusive, safe to run | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, badsecrets, bevigil, binaryedge, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, emailformat, filedownload, fingerprintx, fullhunt, git, github, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, leakix, myssl, nsec, ntlm, oauth, otx, passivetotal, pgp, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomain_hijack, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | +| passive | 50 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, github, hackertarget, hunterio, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, nsec, otx, passivetotal, pgp, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | +| subdomain-enum | 44 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, github, hackertarget, httpx, hunterio, ipneighbor, leakix, massdns, myssl, nsec, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomain_hijack, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | +| active | 38 | Makes active connections to target systems | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, dnszonetransfer, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, social, sslcert, subdomain_hijack, telerik, url_manipulation, vhost, wafw00f, wappalyzer | +| web-thorough | 24 | More advanced web scanning functionality | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, ffuf_shortnames, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, wappalyzer | +| aggressive | 18 | Generates a large amount of network traffic | bypass403, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | +| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, badsecrets, bucket_aws, bucket_azure, bucket_firebase, bucket_gcp, git, httpx, hunt, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, subdomain_hijack, wappalyzer | +| cloud-enum | 10 | Enumerates cloud resources | azure_realm, azure_tenant, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, httpx, oauth, subdomain_hijack | +| slow | 9 | May take a long time to complete | bucket_digitalocean, fingerprintx, massdns, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, vhost | +| affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | +| email-enum | 5 | Enumerates email addresses | emailformat, hunterio, pgp, skymem, sslcert | +| deadly | 3 | Highly aggressive | ffuf, nuclei, vhost | +| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | +| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | +| portscan | 2 | Discovers open ports | masscan, nmap | +| report | 2 | Generates a report at the end of the scan | affiliates, asn | +| social-enum | 2 | Enumerates social media | httpx, social | +| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | +| subdomain-hijack | 1 | Detects hijackable subdomains | subdomain_hijack | +| web-screenshots | 1 | Takes screenshots of web pages | gowitness | ## Dependencies From 6b4a3210dcaf6a4f871698d6750b4db0f19d6dd7 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 13 Oct 2023 10:35:01 -0400 Subject: [PATCH 18/49] small bugfix for asset_inventory --- bbot/modules/output/asset_inventory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index d249ae1cd..56e94aaa2 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -92,7 +92,7 @@ def sort_key(asset): ports = getattr(asset, "ports", set()) ports = [str(p) for p in sorted([int(p) for p in asset.ports])] ips = sorted([str(i) for i in getattr(asset, "ip_addresses", [])]) - host = getattr(asset, "host", "") + host = self.helpers.make_ip_type(getattr(asset, "host", "")) if host and isinstance(host, str): _, domain = self.helpers.split_domain(host) if domain: @@ -197,7 +197,7 @@ def __init__(self, host): def absorb_csv_row(self, row): # host - host = row.get("Host", "").strip() + host = make_ip_type(row.get("Host", "").strip()) if host and not is_ip(host): self.host = host # ips From 0bad64895c08d0071b37918dd034486d37161f43 Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Fri, 13 Oct 2023 15:01:47 +0000 Subject: [PATCH 19/49] Refresh module docs --- README.md | 207 +++++++++++++++---------------- docs/modules/list_of_modules.md | 208 ++++++++++++++++---------------- docs/scanning/advanced.md | 16 ++- docs/scanning/configuration.md | 5 + docs/scanning/events.md | 53 ++++---- docs/scanning/index.md | 44 +++---- 6 files changed, 276 insertions(+), 257 deletions(-) diff --git a/README.md b/README.md index f97b22955..2c8ff90f5 100644 --- a/README.md +++ b/README.md @@ -189,106 +189,109 @@ Special thanks to the following people who made BBOT possible: ## List of BBOT Modules -| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | -|----------------------|----------|-----------------|------------------------------------------------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------| -| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | -| bucket_aws | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_gcp | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | -| dnszonetransfer | scan | No | Attempt DNS zone transfers | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | -| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | -| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | -| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | -| git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | -| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | -| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | -| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | -| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | -| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic, web-thorough | URL | URL_HINT | -| masscan | scan | No | Port scan IP subnets with masscan | active, aggressive, portscan | SCAN | OPEN_TCP_PORT | -| nmap | scan | No | Execute port scans with nmap | active, aggressive, portscan, web-thorough | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT | -| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic, web-thorough | HTTP_RESPONSE, URL | DNS_NAME, FINDING | -| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, VULNERABILITY | -| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | -| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | -| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | -| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | -| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic, web-thorough | URL | URL_UNVERIFIED | -| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | -| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | -| social | scan | No | Look for social media links in webpages | active, safe, social-enum | URL_UNVERIFIED | SOCIAL | -| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | -| subdomain_hijack | scan | No | Detect hijackable subdomains | active, cloud-enum, safe, subdomain-enum, subdomain-hijack, web-basic, web-thorough | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING | -| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, slow, web-thorough | URL | FINDING, VULNERABILITY | -| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | -| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | -| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | -| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic, web-thorough | HTTP_RESPONSE | TECHNOLOGY | -| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | -| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | -| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | -| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | -| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| crobat | scan | No | Query Project Crobat for subdomains | passive, safe | DNS_NAME | DNS_NAME | -| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| dnscommonsrv | scan | No | Check for common SRV records | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | -| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| github | scan | Yes | Query Github's API for related repositories | passive, safe, subdomain-enum | DNS_NAME | URL_UNVERIFIED | -| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | -| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | -| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | -| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | -| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| massdns | scan | No | Brute-force subdomains with massdns (highly effective) | aggressive, passive, slow, subdomain-enum | DNS_NAME | DNS_NAME | -| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| nsec | scan | No | Enumerate subdomains by NSEC-walking | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | -| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| riddler | scan | No | Query riddler.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | -| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| sublist3r | scan | No | Query sublist3r's API for subdomains | passive, safe | DNS_NAME | DNS_NAME | -| threatminer | scan | No | Query threatminer's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | -| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | -| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | -| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| asset_inventory | output | No | Output to an asset inventory style flattened CSV file | | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY | IP_ADDRESS, OPEN_TCP_PORT | -| csv | output | No | Output to CSV | | * | | -| discord | output | No | Message a Discord channel when certain events are encountered | | * | | -| http | output | No | Send every event to a custom URL via a web request | | * | | -| human | output | No | Output to text | | * | | -| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | -| neo4j | output | No | Output to Neo4j | | * | | -| python | output | No | Output via Python API | | * | | -| slack | output | No | Message a Slack channel when certain events are encountered | | * | | -| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | -| teams | output | No | Message a Slack channel when certain events are encountered | | * | | -| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | -| websocket | output | No | Output to websockets | | * | | -| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | -| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED | -| speculate | internal | No | Derive certain event types from others by common sense | passive | DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, STORAGE_BUCKET, URL, URL_UNVERIFIED | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT | +| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | +|----------------------|----------|-----------------|-----------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|----------------------------------------------------| +| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | +| bucket_aws | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_gcp | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | +| dnszonetransfer | scan | No | Attempt DNS zone transfers | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | +| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | +| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, safe | HTTP_RESPONSE, URL_UNVERIFIED | | +| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | +| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | +| git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | +| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | +| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | +| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | +| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | +| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic, web-thorough | URL | URL_HINT | +| masscan | scan | No | Port scan IP subnets with masscan | active, aggressive, portscan | SCAN | OPEN_TCP_PORT | +| nmap | scan | No | Execute port scans with nmap | active, aggressive, portscan, web-thorough | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT | +| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic, web-thorough | HTTP_RESPONSE, URL | DNS_NAME, FINDING | +| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, VULNERABILITY | +| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | +| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic, web-thorough | URL | URL_UNVERIFIED | +| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | +| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | +| social | scan | No | Look for social media links in webpages | active, safe, social-enum | URL_UNVERIFIED | SOCIAL | +| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | +| subdomain_hijack | scan | No | Detect hijackable subdomains | active, cloud-enum, safe, subdomain-enum, subdomain-hijack, web-basic, web-thorough | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING | +| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, slow, web-thorough | URL | FINDING, VULNERABILITY | +| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | +| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | +| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | +| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic, web-thorough | HTTP_RESPONSE | TECHNOLOGY | +| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | +| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | +| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | +| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| credshed | scan | Yes | Send queries to your own credshed server to check for known credentials of your targets | passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | +| crobat | scan | No | Query Project Crobat for subdomains | passive, safe | DNS_NAME | DNS_NAME | +| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dehashed | scan | Yes | Execute queries against dehashed.com for exposed credentials | passive | DNS_NAME | HASHED_PASSWORD, PASSWORD, USERNAME | +| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dnscommonsrv | scan | No | Check for common SRV records | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| github | scan | Yes | Query Github's API for related repositories | passive, safe, subdomain-enum | DNS_NAME | URL_UNVERIFIED | +| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | +| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | +| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | +| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | +| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| massdns | scan | No | Brute-force subdomains with massdns (highly effective) | aggressive, passive, slow, subdomain-enum | DNS_NAME | DNS_NAME | +| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| nsec | scan | No | Enumerate subdomains by NSEC-walking | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| riddler | scan | No | Query riddler.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| sublist3r | scan | No | Query sublist3r's API for subdomains | passive, safe | DNS_NAME | DNS_NAME | +| threatminer | scan | No | Query threatminer's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | +| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| asset_inventory | output | No | Output to an asset inventory style flattened CSV file | | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY | IP_ADDRESS, OPEN_TCP_PORT | +| csv | output | No | Output to CSV | | * | | +| discord | output | No | Message a Discord channel when certain events are encountered | | * | | +| http | output | No | Send every event to a custom URL via a web request | | * | | +| human | output | No | Output to text | | * | | +| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | +| neo4j | output | No | Output to Neo4j | | * | | +| python | output | No | Output via Python API | | * | | +| slack | output | No | Message a Slack channel when certain events are encountered | | * | | +| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | +| teams | output | No | Message a Slack channel when certain events are encountered | | * | | +| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | +| websocket | output | No | Output to websockets | | * | | +| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | +| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED | +| speculate | internal | No | Derive certain event types from others by common sense | passive | DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, STORAGE_BUCKET, URL, URL_UNVERIFIED | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT | diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 9a4e23da5..177bc21dc 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -1,109 +1,111 @@ # List of Modules -| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | -|----------------------|----------|-----------------|------------------------------------------------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------| -| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | -| bucket_aws | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bucket_gcp | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | -| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | -| dnszonetransfer | scan | No | Attempt DNS zone transfers | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | -| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | -| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, safe | HTTP_RESPONSE, URL_UNVERIFIED | | -| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | -| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | -| git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | -| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | -| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | -| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | -| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | -| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic, web-thorough | URL | URL_HINT | -| masscan | scan | No | Port scan IP subnets with masscan | active, aggressive, portscan | SCAN | OPEN_TCP_PORT | -| nmap | scan | No | Execute port scans with nmap | active, aggressive, portscan, web-thorough | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT | -| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic, web-thorough | HTTP_RESPONSE, URL | DNS_NAME, FINDING | -| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, VULNERABILITY | -| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | -| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | -| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | -| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | -| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic, web-thorough | URL | URL_UNVERIFIED | -| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | -| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | -| social | scan | No | Look for social media links in webpages | active, safe, social-enum | URL_UNVERIFIED | SOCIAL | -| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | -| subdomain_hijack | scan | No | Detect hijackable subdomains | active, cloud-enum, safe, subdomain-enum, subdomain-hijack, web-basic, web-thorough | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING | -| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, slow, web-thorough | URL | FINDING, VULNERABILITY | -| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | -| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | -| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | -| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic, web-thorough | HTTP_RESPONSE | TECHNOLOGY | -| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | -| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | -| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | -| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | -| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| crobat | scan | No | Query Project Crobat for subdomains | passive, safe | DNS_NAME | DNS_NAME | -| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| dnscommonsrv | scan | No | Check for common SRV records | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | -| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| github | scan | Yes | Query Github's API for related repositories | passive, safe, subdomain-enum | DNS_NAME | URL_UNVERIFIED | -| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | -| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | -| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | -| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | -| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| massdns | scan | No | Brute-force subdomains with massdns (highly effective) | aggressive, passive, slow, subdomain-enum | DNS_NAME | DNS_NAME | -| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| nsec | scan | No | Enumerate subdomains by NSEC-walking | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | -| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| riddler | scan | No | Query riddler.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | -| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| sublist3r | scan | No | Query sublist3r's API for subdomains | passive, safe | DNS_NAME | DNS_NAME | -| threatminer | scan | No | Query threatminer's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | -| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | -| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | -| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | -| asset_inventory | output | No | Output to an asset inventory style flattened CSV file | | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY | IP_ADDRESS, OPEN_TCP_PORT | -| csv | output | No | Output to CSV | | * | | -| discord | output | No | Message a Discord channel when certain events are encountered | | * | | -| http | output | No | Send every event to a custom URL via a web request | | * | | -| human | output | No | Output to text | | * | | -| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | -| neo4j | output | No | Output to Neo4j | | * | | -| python | output | No | Output via Python API | | * | | -| slack | output | No | Message a Slack channel when certain events are encountered | | * | | -| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | -| teams | output | No | Message a Slack channel when certain events are encountered | | * | | -| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | -| websocket | output | No | Output to websockets | | * | | -| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | -| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED | -| speculate | internal | No | Derive certain event types from others by common sense | passive | DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, STORAGE_BUCKET, URL, URL_UNVERIFIED | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT | +| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | +|----------------------|----------|-----------------|-----------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------|----------------------------------------------------| +| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | +| bucket_aws | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bucket_gcp | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | +| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | +| dnszonetransfer | scan | No | Attempt DNS zone transfers | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | +| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | +| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, safe | HTTP_RESPONSE, URL_UNVERIFIED | | +| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | +| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | +| git | scan | No | Check for exposed .git repositories | active, safe, web-basic, web-thorough | URL | FINDING | +| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | +| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | +| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | +| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | +| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic, web-thorough | URL | URL_HINT | +| masscan | scan | No | Port scan IP subnets with masscan | active, aggressive, portscan | SCAN | OPEN_TCP_PORT | +| nmap | scan | No | Execute port scans with nmap | active, aggressive, portscan, web-thorough | DNS_NAME, IP_ADDRESS | OPEN_TCP_PORT | +| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic, web-thorough | HTTP_RESPONSE, URL | DNS_NAME, FINDING | +| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, VULNERABILITY | +| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | +| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE | FINDING | +| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic, web-thorough | URL | URL_UNVERIFIED | +| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic, web-thorough | HTTP_RESPONSE | FINDING | +| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | +| social | scan | No | Look for social media links in webpages | active, safe, social-enum | URL_UNVERIFIED | SOCIAL | +| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic, web-thorough | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | +| subdomain_hijack | scan | No | Detect hijackable subdomains | active, cloud-enum, safe, subdomain-enum, subdomain-hijack, web-basic, web-thorough | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING | +| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, slow, web-thorough | URL | FINDING, VULNERABILITY | +| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | +| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | +| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | +| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic, web-thorough | HTTP_RESPONSE | TECHNOLOGY | +| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | +| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | +| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | +| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| credshed | scan | Yes | Send queries to your own credshed server to check for known credentials of your targets | passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | +| crobat | scan | No | Query Project Crobat for subdomains | passive, safe | DNS_NAME | DNS_NAME | +| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dehashed | scan | Yes | Execute queries against dehashed.com for exposed credentials | passive | DNS_NAME | HASHED_PASSWORD, PASSWORD, USERNAME | +| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dnscommonsrv | scan | No | Check for common SRV records | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| github | scan | Yes | Query Github's API for related repositories | passive, safe, subdomain-enum | DNS_NAME | URL_UNVERIFIED | +| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | +| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | +| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | +| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | +| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| massdns | scan | No | Brute-force subdomains with massdns (highly effective) | aggressive, passive, slow, subdomain-enum | DNS_NAME | DNS_NAME | +| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| nsec | scan | No | Enumerate subdomains by NSEC-walking | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| riddler | scan | No | Query riddler.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | +| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| sublist3r | scan | No | Query sublist3r's API for subdomains | passive, safe | DNS_NAME | DNS_NAME | +| threatminer | scan | No | Query threatminer's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | +| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | +| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | +| asset_inventory | output | No | Output to an asset inventory style flattened CSV file | | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY | IP_ADDRESS, OPEN_TCP_PORT | +| csv | output | No | Output to CSV | | * | | +| discord | output | No | Message a Discord channel when certain events are encountered | | * | | +| http | output | No | Send every event to a custom URL via a web request | | * | | +| human | output | No | Output to text | | * | | +| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | +| neo4j | output | No | Output to Neo4j | | * | | +| python | output | No | Output via Python API | | * | | +| slack | output | No | Message a Slack channel when certain events are encountered | | * | | +| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | +| teams | output | No | Message a Slack channel when certain events are encountered | | * | | +| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | +| websocket | output | No | Output to websockets | | * | | +| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | +| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE | URL_UNVERIFIED | +| speculate | internal | No | Derive certain event types from others by common sense | passive | DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, STORAGE_BUCKET, URL, URL_UNVERIFIED | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT | For a list of module config options, see [Module Options](../scanning/configuration.md#module-config-options). diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 589b08e20..5dac962b6 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -33,10 +33,16 @@ asyncio.run(main()) ```text -usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-m MODULE [MODULE ...]] [-l] - [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] [-o DIR] - [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-config] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] - [--version] +usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] + [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] + [--strict-scope] [-m MODULE [MODULE ...]] [-l] + [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] + [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] + [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] + [-o DIR] [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] + [--dry-run] [--current-config] + [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] + [-a] [--version] Bighuge BLS OSINT Tool @@ -55,7 +61,7 @@ Target: Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] - Modules to enable. Choices: affiliates,anubisdb,asn,azure_realm,azure_tenant,badsecrets,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_digitalocean,bucket_firebase,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,crobat,crt,digitorus,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,github,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,nmap,nsec,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomain_hijack,subdomaincenter,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye + Modules to enable. Choices: affiliates,anubisdb,asn,azure_realm,azure_tenant,badsecrets,bevigil,binaryedge,bucket_aws,bucket_azure,bucket_digitalocean,bucket_firebase,bucket_gcp,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dehashed,digitorus,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,github,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,nmap,nsec,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomain_hijack,subdomaincenter,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye -l, --list-modules List available modules. -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 25403fc02..90282ef66 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -295,6 +295,11 @@ Many modules accept their own configuration options. These options have the abil | modules.censys.api_secret | str | Censys.io API Secret | | | modules.censys.max_pages | int | Maximum number of pages to fetch (100 results per page) | 5 | | modules.chaos.api_key | str | Chaos API key | | +| modules.credshed.credshed_url | str | URL of credshed server | | +| modules.credshed.password | str | Credshed password | | +| modules.credshed.username | str | Credshed username | | +| modules.dehashed.api_key | str | DeHashed API Key | | +| modules.dehashed.username | str | Email Address associated with your API key | | | modules.fullhunt.api_key | str | FullHunt API Key | | | modules.github.api_key | str | Github token | | | modules.hunterio.api_key | str | Hunter.IO API key | | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index fc05eb822..fa1e156cc 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -49,31 +49,34 @@ Below is a full list of event types along with which modules produce/consume the ## List of Event Types -| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | -|---------------------|-----------------------|-----------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| * | 11 | 0 | affiliates, csv, discord, http, human, json, neo4j, python, slack, teams, websocket | | -| ASN | 0 | 1 | | asn | -| DNS_NAME | 54 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, bevigil, binaryedge, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, emailformat, fullhunt, github, hackertarget, hunterio, leakix, massdns, myssl, nmap, nsec, oauth, otx, passivetotal, pgp, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomain_hijack, subdomaincenter, subdomains, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, hackertarget, hunterio, leakix, massdns, myssl, nsec, ntlm, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, sublist3r, threatminer, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | -| DNS_NAME_UNRESOLVED | 3 | 0 | speculate, subdomain_hijack, subdomains | | -| EMAIL_ADDRESS | 0 | 5 | | emailformat, hunterio, pgp, skymem, sslcert | -| FINDING | 2 | 21 | asset_inventory, web_report | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, git, host_header, hunt, ntlm, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, smuggler, speculate, subdomain_hijack, telerik, url_manipulation | -| GEOLOCATION | 0 | 2 | | ip2location, ipstack | -| HTTP_RESPONSE | 12 | 1 | badsecrets, excavate, filedownload, host_header, hunt, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, wappalyzer | httpx | -| IP_ADDRESS | 7 | 3 | asn, asset_inventory, ip2location, ipneighbor, ipstack, nmap, speculate | asset_inventory, ipneighbor, speculate | -| IP_RANGE | 1 | 0 | speculate | | -| OPEN_TCP_PORT | 4 | 4 | asset_inventory, fingerprintx, httpx, sslcert | asset_inventory, masscan, nmap, speculate | -| PROTOCOL | 0 | 1 | | fingerprintx | -| SCAN | 1 | 0 | masscan | | -| SOCIAL | 0 | 1 | | social | -| STORAGE_BUCKET | 6 | 5 | bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, speculate | bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp | -| TECHNOLOGY | 2 | 2 | asset_inventory, web_report | gowitness, wappalyzer | -| URL | 18 | 2 | asset_inventory, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | -| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | -| URL_UNVERIFIED | 5 | 11 | filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, excavate, ffuf, ffuf_shortnames, github, gowitness, hunterio, robots, urlscan, wayback | -| VHOST | 1 | 1 | web_report | vhost | -| VULNERABILITY | 2 | 4 | asset_inventory, web_report | badsecrets, generic_ssrf, nuclei, telerik | -| WAF | 0 | 1 | | wafw00f | -| WEBSCREENSHOT | 0 | 1 | | gowitness | +| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | +|---------------------|-----------------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| * | 11 | 0 | affiliates, csv, discord, http, human, json, neo4j, python, slack, teams, websocket | | +| ASN | 0 | 1 | | asn | +| DNS_NAME | 56 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, bevigil, binaryedge, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, emailformat, fullhunt, github, hackertarget, hunterio, leakix, massdns, myssl, nmap, nsec, oauth, otx, passivetotal, pgp, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomain_hijack, subdomaincenter, subdomains, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, hackertarget, hunterio, leakix, massdns, myssl, nsec, ntlm, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, sublist3r, threatminer, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | +| DNS_NAME_UNRESOLVED | 3 | 0 | speculate, subdomain_hijack, subdomains | | +| EMAIL_ADDRESS | 0 | 6 | | credshed, emailformat, hunterio, pgp, skymem, sslcert | +| FINDING | 2 | 21 | asset_inventory, web_report | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, git, host_header, hunt, ntlm, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, smuggler, speculate, subdomain_hijack, telerik, url_manipulation | +| GEOLOCATION | 0 | 2 | | ip2location, ipstack | +| HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | +| HTTP_RESPONSE | 12 | 1 | badsecrets, excavate, filedownload, host_header, hunt, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, wappalyzer | httpx | +| IP_ADDRESS | 7 | 3 | asn, asset_inventory, ip2location, ipneighbor, ipstack, nmap, speculate | asset_inventory, ipneighbor, speculate | +| IP_RANGE | 1 | 0 | speculate | | +| OPEN_TCP_PORT | 4 | 4 | asset_inventory, fingerprintx, httpx, sslcert | asset_inventory, masscan, nmap, speculate | +| PASSWORD | 0 | 2 | | credshed, dehashed | +| PROTOCOL | 0 | 1 | | fingerprintx | +| SCAN | 1 | 0 | masscan | | +| SOCIAL | 0 | 1 | | social | +| STORAGE_BUCKET | 6 | 5 | bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, speculate | bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp | +| TECHNOLOGY | 2 | 2 | asset_inventory, web_report | gowitness, wappalyzer | +| URL | 18 | 2 | asset_inventory, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | +| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | +| URL_UNVERIFIED | 5 | 11 | filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, excavate, ffuf, ffuf_shortnames, github, gowitness, hunterio, robots, urlscan, wayback | +| USERNAME | 0 | 2 | | credshed, dehashed | +| VHOST | 1 | 1 | web_report | vhost | +| VULNERABILITY | 2 | 4 | asset_inventory, web_report | badsecrets, generic_ssrf, nuclei, telerik | +| WAF | 0 | 1 | | wafw00f | +| WEBSCREENSHOT | 0 | 1 | | gowitness | ## Findings Vs. Vulnerabilties diff --git a/docs/scanning/index.md b/docs/scanning/index.md index 2be84977f..b17da03de 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -107,28 +107,28 @@ A single module can have multiple flags. For example, the `securitytrails` modul ### List of Flags -| Flag | # Modules | Description | Modules | -|------------------|-------------|-----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| safe | 68 | Non-intrusive, safe to run | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, badsecrets, bevigil, binaryedge, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, emailformat, filedownload, fingerprintx, fullhunt, git, github, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, leakix, myssl, nsec, ntlm, oauth, otx, passivetotal, pgp, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomain_hijack, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | -| passive | 50 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, github, hackertarget, hunterio, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, nsec, otx, passivetotal, pgp, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 44 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, github, hackertarget, httpx, hunterio, ipneighbor, leakix, massdns, myssl, nsec, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomain_hijack, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | -| active | 38 | Makes active connections to target systems | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, dnszonetransfer, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, social, sslcert, subdomain_hijack, telerik, url_manipulation, vhost, wafw00f, wappalyzer | -| web-thorough | 24 | More advanced web scanning functionality | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, ffuf_shortnames, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, wappalyzer | -| aggressive | 18 | Generates a large amount of network traffic | bypass403, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | -| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, badsecrets, bucket_aws, bucket_azure, bucket_firebase, bucket_gcp, git, httpx, hunt, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, subdomain_hijack, wappalyzer | -| cloud-enum | 10 | Enumerates cloud resources | azure_realm, azure_tenant, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, httpx, oauth, subdomain_hijack | -| slow | 9 | May take a long time to complete | bucket_digitalocean, fingerprintx, massdns, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, vhost | -| affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | -| email-enum | 5 | Enumerates email addresses | emailformat, hunterio, pgp, skymem, sslcert | -| deadly | 3 | Highly aggressive | ffuf, nuclei, vhost | -| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | -| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | -| portscan | 2 | Discovers open ports | masscan, nmap | -| report | 2 | Generates a report at the end of the scan | affiliates, asn | -| social-enum | 2 | Enumerates social media | httpx, social | -| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | -| subdomain-hijack | 1 | Detects hijackable subdomains | subdomain_hijack | -| web-screenshots | 1 | Takes screenshots of web pages | gowitness | +| Flag | # Modules | Description | Modules | +|------------------|-------------|-----------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| safe | 69 | Non-intrusive, safe to run | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, badsecrets, bevigil, binaryedge, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, emailformat, filedownload, fingerprintx, fullhunt, git, github, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, ip2location, ipstack, leakix, myssl, nsec, ntlm, oauth, otx, passivetotal, pgp, rapiddns, riddler, robots, secretsdb, securitytrails, shodan_dns, sitedossier, skymem, social, sslcert, subdomain_hijack, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | +| passive | 52 | Never connects to target systems | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, github, hackertarget, hunterio, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, nsec, otx, passivetotal, pgp, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye | +| subdomain-enum | 44 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, github, hackertarget, httpx, hunterio, ipneighbor, leakix, massdns, myssl, nsec, oauth, otx, passivetotal, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomain_hijack, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye | +| active | 38 | Makes active connections to target systems | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, dnszonetransfer, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, social, sslcert, subdomain_hijack, telerik, url_manipulation, vhost, wafw00f, wappalyzer | +| web-thorough | 24 | More advanced web scanning functionality | badsecrets, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, bypass403, ffuf_shortnames, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, wappalyzer | +| aggressive | 18 | Generates a large amount of network traffic | bypass403, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f | +| web-basic | 17 | Basic, non-intrusive web scan functionality | azure_realm, badsecrets, bucket_aws, bucket_azure, bucket_firebase, bucket_gcp, git, httpx, hunt, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, subdomain_hijack, wappalyzer | +| cloud-enum | 10 | Enumerates cloud resources | azure_realm, azure_tenant, bucket_aws, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_gcp, httpx, oauth, subdomain_hijack | +| slow | 9 | May take a long time to complete | bucket_digitalocean, fingerprintx, massdns, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, vhost | +| affiliates | 8 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, viewdns, zoomeye | +| email-enum | 5 | Enumerates email addresses | emailformat, hunterio, pgp, skymem, sslcert | +| deadly | 3 | Highly aggressive | ffuf, nuclei, vhost | +| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | +| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | +| portscan | 2 | Discovers open ports | masscan, nmap | +| report | 2 | Generates a report at the end of the scan | affiliates, asn | +| social-enum | 2 | Enumerates social media | httpx, social | +| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | +| subdomain-hijack | 1 | Detects hijackable subdomains | subdomain_hijack | +| web-screenshots | 1 | Takes screenshots of web pages | gowitness | ## Dependencies From eb08f7e4f1890ac61d4be5090045d25cab3883e6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 13 Oct 2023 11:31:55 -0400 Subject: [PATCH 20/49] bumped version 1.1.1 --> 1.1.2 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ee2a01c94..5ddb60573 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ livereload = "^2.6.3" env = [ "BBOT_TESTING = True", "PYTHONASYNCIODEBUG = 1" -] +] [build-system] requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] @@ -89,7 +89,7 @@ line-length = 119 [tool.poetry-dynamic-versioning] enable = true metadata = false -format-jinja = 'v1.1.1{% if branch == "dev" %}.{{ distance }}rc{% endif %}' +format-jinja = 'v1.1.2{% if branch == "dev" %}.{{ distance }}rc{% endif %}' [tool.poetry-dynamic-versioning.substitution] files = ["*/__init__.py"] From 5170e874866772292274c6035b8e815379446b55 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 15 Oct 2023 17:29:18 -0400 Subject: [PATCH 21/49] excavate CSP submodule functional --- bbot/modules/internal/excavate.py | 25 ++++++++++++++++--- .../module_tests/test_module_excavate.py | 12 +++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 51b8a4dc2..56cb92625 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -4,7 +4,7 @@ import jwt as j from urllib.parse import urljoin -from bbot.core.helpers.regexes import _email_regex +from bbot.core.helpers.regexes import _email_regex, dns_name_regex from bbot.modules.internal.base import BaseInternalModule @@ -36,6 +36,25 @@ def report(self, result, name, event): pass +class CSPExtractor(BaseExtractor): + regexes = {"CSP": r"(?i)(?m)Content-Security-Policy:.+$"} + + def extract_domains(self, csp): + domains = dns_name_regex.findall(csp) + unique_domains = set(domains) + return unique_domains + + async def search(self, content, event, **kwargs): + results = set() + async for csp, name in self._search(content, event, **kwargs): + extracted_domains = self.extract_domains(csp) + for domain in extracted_domains: + self.report(domain, event, **kwargs) + + def report(self, domain, event, **kwargs): + self.excavate.emit_event(domain, "DNS_NAME_UNRESOLVED", source=event) + + class HostnameExtractor(BaseExtractor): regexes = {} @@ -297,6 +316,7 @@ class excavate(BaseInternalModule): scope_distance_modifier = None async def setup(self): + self.csp = CSPExtractor(self) self.hostname = HostnameExtractor(self) self.url = URLExtractor(self) self.email = EmailExtractor(self) @@ -306,7 +326,6 @@ async def setup(self): self.serialization = SerializationExtractor(self) self.functionality = FunctionalityExtractor(self) self.max_redirects = self.scan.config.get("http_max_redirects", 5) - return True async def search(self, source, extractors, event, **kwargs): @@ -369,7 +388,7 @@ async def handle_event(self, event): headers = self.helpers.recursive_decode(event.data.get("raw_header", "")) await self.search( headers, - [self.hostname, self.url, self.email, self.error_extractor, self.jwt, self.serialization], + [self.hostname, self.url, self.email, self.error_extractor, self.jwt, self.serialization, self.csp], event, consider_spider_danger=False, ) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 1d74aa2ba..18d453b03 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -209,3 +209,15 @@ def check(self, module_test, events): url_data = [e.data for e in url_events if "spider-danger" not in e.tags] assert "http://127.0.0.1:8888/10" in url_data assert "http://127.0.0.1:8888/11" not in url_data + + +class TestExcavateCSP(TestExcavate): + csp_test_header = "default-src 'self'; script-src fake.domain.com; object-src 'none';" + + async def setup_before_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"headers": {"Content-Security-Policy": self.csp_test_header}} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + assert any(e.data == "fake.domain.com" for e in events) From 864c933f166b9c221830f23a2f8c55ab0b644134 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 15 Oct 2023 17:36:41 -0400 Subject: [PATCH 22/49] removed uncessary set --- bbot/modules/internal/excavate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 56cb92625..296cb5a59 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -45,7 +45,6 @@ def extract_domains(self, csp): return unique_domains async def search(self, content, event, **kwargs): - results = set() async for csp, name in self._search(content, event, **kwargs): extracted_domains = self.extract_domains(csp) for domain in extracted_domains: From 4ee94ed74bc2bbb7b5390a6cc7441039ca938f98 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 11:47:01 -0400 Subject: [PATCH 23/49] added missing dev docs --- docs/dev/helpers/command.md | 13 +++++++++++++ docs/dev/helpers/dns.md | 19 +++++++++++++++++++ docs/dev/helpers/interactsh.md | 5 +++++ docs/dev/helpers/web.md | 15 +++++++++++++++ docs/dev/helpers/wordcloud.md | 13 +++++++++++++ 5 files changed, 65 insertions(+) create mode 100644 docs/dev/helpers/command.md create mode 100644 docs/dev/helpers/dns.md create mode 100644 docs/dev/helpers/interactsh.md create mode 100644 docs/dev/helpers/web.md create mode 100644 docs/dev/helpers/wordcloud.md diff --git a/docs/dev/helpers/command.md b/docs/dev/helpers/command.md new file mode 100644 index 000000000..a3da89c08 --- /dev/null +++ b/docs/dev/helpers/command.md @@ -0,0 +1,13 @@ +# Command Helpers + +These are helpers related to executing shell commands. They are used throughout BBOT and its modules for executing various binaries such as `nmap`, `nuclei`, etc. + +Note that these helpers can be invoked directly from `self.helpers`, e.g.: + +```python +self.helpers.run("ls", "-l") +``` + +::: bbot.core.helpers.command + options: + show_root_heading: false diff --git a/docs/dev/helpers/dns.md b/docs/dev/helpers/dns.md new file mode 100644 index 000000000..5a51d6116 --- /dev/null +++ b/docs/dev/helpers/dns.md @@ -0,0 +1,19 @@ +# DNS + +These are helpers related to DNS resolution. They are used throughout BBOT and its modules for performing DNS lookups and detecting DNS wildcards, etc. + +Note that these helpers can be invoked directly from `self.helpers`, e.g.: + +```python +self.helpers.resolve("evilcorp.com") +``` + +::: bbot.core.helpers.dns.DNSHelper + handler: python + options: + members: + - resolve + - resolve_batch + - resolve_raw + - is_wildcard + - is_wildcard_domain diff --git a/docs/dev/helpers/interactsh.md b/docs/dev/helpers/interactsh.md new file mode 100644 index 000000000..5431e6337 --- /dev/null +++ b/docs/dev/helpers/interactsh.md @@ -0,0 +1,5 @@ +# Interact.sh + +::: bbot.core.helpers.interactsh.Interactsh + options: + show_root_heading: false diff --git a/docs/dev/helpers/web.md b/docs/dev/helpers/web.md new file mode 100644 index 000000000..b25f0e7ca --- /dev/null +++ b/docs/dev/helpers/web.md @@ -0,0 +1,15 @@ +# Web + +These are helpers for making various web requests. + +Note that these helpers can be invoked directly from `self.helpers`, e.g.: + +```python +self.helpers.request("https://www.evilcorp.com") +``` + +::: bbot.core.helpers.web + options: + show_root_heading: false + members: + - WebHelper diff --git a/docs/dev/helpers/wordcloud.md b/docs/dev/helpers/wordcloud.md new file mode 100644 index 000000000..cc2d6671c --- /dev/null +++ b/docs/dev/helpers/wordcloud.md @@ -0,0 +1,13 @@ +# Word Cloud + +These are helpers related to BBOT's Word Cloud, a mechanism for storing target-specific keywords that are useful for custom wordlists, etc. + +Note that these helpers can be invoked directly from `self.helpers`, e.g.: + +```python +self.helpers.word_cloud +``` + +::: bbot.core.helpers.wordcloud + options: + show_root_heading: false From 84c2dc4df8a0e094e837d2ecc2a195d49bbe82fa Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 15:53:01 -0400 Subject: [PATCH 24/49] fix console hang --- bbot/cli.py | 48 +++++++++++++++++++++++++++--------------------- poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 4c45254d6..b2df0c9b9 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -5,8 +5,8 @@ import sys import asyncio import logging -import threading import traceback +from aioconsole import ainput from omegaconf import OmegaConf from contextlib import suppress @@ -306,36 +306,42 @@ async def _main(): log.hugesuccess(f"Scan ready. Press enter to execute {scanner.name}") input() - def keyboard_listen(): - allowed_errors = 10 + def handle_keyboard_input(keyboard_input): kill_regex = re.compile(r"kill (?P[a-z0-9_]+)") + if keyboard_input: + log.verbose(f'Got keyboard input: "{keyboard_input}"') + kill_match = kill_regex.match(keyboard_input) + if kill_match: + module = kill_match.group("module") + if module in scanner.modules: + log.hugewarning(f'Killing module: "{module}"') + scanner.manager.kill_module(module, message="killed by user") + else: + log.warning(f'Invalid module: "{module}"') + else: + toggle_log_level(logger=log) + scanner.manager.modules_status(_log=True) + + async def akeyboard_listen(): + allowed_errors = 10 while 1: keyboard_input = "a" try: - keyboard_input = input() - allowed_errors = 10 + keyboard_input = await ainput() except Exception: allowed_errors -= 1 - if keyboard_input: - log.verbose(f'Got keyboard input: "{keyboard_input}"') - kill_match = kill_regex.match(keyboard_input) - if kill_match: - module = kill_match.group("module") - if module in scanner.modules: - log.hugewarning(f'Killing module: "{module}"') - scanner.manager.kill_module(module, message="killed by user") - else: - log.warning(f'Invalid module: "{module}"') - else: - toggle_log_level(logger=log) - scanner.manager.modules_status(_log=True) + handle_keyboard_input(keyboard_input) if allowed_errors <= 0: break - keyboard_listen_thread = threading.Thread(target=keyboard_listen, daemon=True) - keyboard_listen_thread.start() + try: + keyboard_listen_task = asyncio.create_task(akeyboard_listen()) - await scanner.async_start_without_generator() + await scanner.async_start_without_generator() + finally: + keyboard_listen_task.cancel() + with suppress(asyncio.CancelledError): + await keyboard_listen_task except bbot.core.errors.ScanError as e: log_to_stderr(str(e), level="ERROR") diff --git a/poetry.lock b/poetry.lock index 55c25b105..8d407cff4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +[[package]] +name = "aioconsole" +version = "0.6.2" +description = "Asynchronous console and interfaces for asyncio" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aioconsole-0.6.2-py3-none-any.whl", hash = "sha256:1968021eb03b88fcdf5f5398154b21585e941a7b98c9fcef51c4bb0158156619"}, + {file = "aioconsole-0.6.2.tar.gz", hash = "sha256:bac11286f1062613d2523ceee1ba81c676cd269812b865b66b907448a7b5f63e"}, +] + [[package]] name = "ansible" version = "7.7.0" @@ -2433,4 +2444,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "2215f588e30cd553c593079522ce8b98beb344554688e62d2409237fd245f21d" +content-hash = "f1d4ab56691dfb6c47d2f31ad29c14c74d4f768b2f100d60fccc8167707426f0" diff --git a/pyproject.toml b/pyproject.toml index ee2a01c94..2c4bd2680 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ httpx = {extras = ["http2"], version = "^0.24.1"} dnspython = "^2.4.2" anyio = "4.0.0rc1" httpcore = "^0.17.3" +aioconsole = "^0.6.2" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" From 1655bc7add5d95a7476879dcef4702c3deca6955 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 18:21:15 -0400 Subject: [PATCH 25/49] add microsoft on-prem subdomains --- bbot/modules/base.py | 5 +- bbot/modules/massdns.py | 10 ++- bbot/wordlists/ms_on_prem_subdomains.txt | 101 +++++++++++++++++++++++ 3 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 bbot/wordlists/ms_on_prem_subdomains.txt diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 65731d7fa..9101c800a 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -5,8 +5,8 @@ from contextlib import suppress from ..core.helpers.misc import get_size # noqa +from ..core.errors import ValidationError from ..core.helpers.async_helpers import TaskCounter -from ..core.errors import ValidationError, WordlistError class BaseModule: @@ -527,9 +527,6 @@ async def _setup(self): self.debug(f"Finished setting up module {self.name}") except Exception as e: self.set_error_state() - # soft-fail if it's only a wordlist error - if isinstance(e, WordlistError): - status = None msg = f"{e}" self.trace() return self.name, status, str(msg) diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 7e4331f5b..bb28f045f 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -74,6 +74,12 @@ async def setup(self): self.mutations_tried = set() self.source_events = self.helpers.make_target() self.subdomain_file = await self.helpers.wordlist(self.config.get("wordlist")) + self.subdomain_list = set(self.helpers.read_file(self.subdomain_file)) + + ms_on_prem_string_file = self.helpers.wordlist_dir / "ms_on_prem_subdomains.txt" + ms_on_prem_strings = set(self.helpers.read_file(ms_on_prem_string_file)) + self.subdomain_list.update(ms_on_prem_strings) + self.max_resolvers = self.config.get("max_resolvers", 1000) self.max_mutations = self.config.get("max_mutations", 500) nameservers_url = ( @@ -104,7 +110,7 @@ async def handle_event(self, event): self.source_events.add_target(event) self.info(f"Brute-forcing subdomains for {query} (source: {event.data})") - for hostname in await self.massdns(query, self.helpers.read_file(self.subdomain_file)): + for hostname in await self.massdns(query, self.subdomain_list): self.emit_result(hostname, event, query) def abort_if(self, event): @@ -278,7 +284,7 @@ async def _massdns(self, domain, subdomains): hosts_yielded.add(hostname_hash) yield hostname, data, rdtype - async def finish(self): + async def sfinish(self): found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True) # if we have a lot of rounds to make, don't try mutations on less-populated domains trimmed_found = [] diff --git a/bbot/wordlists/ms_on_prem_subdomains.txt b/bbot/wordlists/ms_on_prem_subdomains.txt new file mode 100644 index 000000000..b323e4605 --- /dev/null +++ b/bbot/wordlists/ms_on_prem_subdomains.txt @@ -0,0 +1,101 @@ +adfs +adfs01 +adfs02 +adfs1 +adfs2 +adfs3 +adfsproxy +adfstest +auth +fed +federate +federated +federation +federationfs +fs +fs1 +fs2 +fs3 +fs4 +gateway +login +portal +saml +sso +sts +wap +webmail +owa +hybrid +hybrid-cloud +email +outlook +exchange +mail2 +webmail2 +mail1 +mailbox +mail01 +mailman +mailgate +mailbackup +mail3 +webmail1 +webmail3 +mailing +mailserver +mailhost +mailer +mailadmin +imap +pop3 +post +post1 +post2 +mail +remote +desktop +desktop1 +desktop2 +desktops +extranet +mydesktop +ra +rdesktop +rdgate +rdp +rdpweb +rds +rdsh +rdweb +remote01 +remote02 +remote1 +remote2 +remote3 +remote4 +remoteapp +remoteapps +remotedesktop +remotegateway +tsweb +vdesktop +vdi +dialin +meet +lync +lyncweb +sip +skype +sfbweb +scheduler +lyncext +lyncdiscoverinternal +access +lyncaccess01 +lyncaccess +lync10 +wac +_sipinternaltls +uc +lyncdiscover From ca8953584daab750ce66bcea31500e36a04fbeff Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 18:21:57 -0400 Subject: [PATCH 26/49] undo debugging changes --- bbot/modules/massdns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index bb28f045f..5dcd10ded 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -284,7 +284,7 @@ async def _massdns(self, domain, subdomains): hosts_yielded.add(hostname_hash) yield hostname, data, rdtype - async def sfinish(self): + async def finish(self): found = sorted(self.found.items(), key=lambda x: len(x[-1]), reverse=True) # if we have a lot of rounds to make, don't try mutations on less-populated domains trimmed_found = [] From 5e3a57a4156beaaf33c147ddaf7711449edcefbc Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 18:40:57 -0400 Subject: [PATCH 27/49] fix ssl verification issues --- bbot/core/helpers/web.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bbot/core/helpers/web.py b/bbot/core/helpers/web.py index f26d4666d..accd55950 100644 --- a/bbot/core/helpers/web.py +++ b/bbot/core/helpers/web.py @@ -132,7 +132,10 @@ class WebHelper: def __init__(self, parent_helper): self.parent_helper = parent_helper self.http_debug = self.parent_helper.config.get("http_debug", False) + self._ssl_context_noverify = None self.ssl_verify = self.parent_helper.config.get("ssl_verify", False) + if self.ssl_verify is False: + self.ssl_verify = self.ssl_context_noverify() self.web_client = self.AsyncClient(persist_cookies=False) def AsyncClient(self, *args, **kwargs): @@ -453,7 +456,7 @@ async def curl(self, *args, **kwargs): curl_command.append("--path-as-is") # respect global ssl verify settings - if self.ssl_verify == False: + if self.ssl_verify is not True: curl_command.append("-k") headers = kwargs.get("headers", {}) @@ -563,13 +566,15 @@ def is_spider_danger(self, source_event, url): return False def ssl_context_noverify(self): - ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3 - ssl_context.set_ciphers("ALL:@SECLEVEL=0") - ssl_context.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option - return ssl_context + if self._ssl_context_noverify is None: + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + ssl_context.options &= ~ssl.OP_NO_SSLv2 & ~ssl.OP_NO_SSLv3 + ssl_context.set_ciphers("ALL:@SECLEVEL=0") + ssl_context.options |= 0x4 # Add the OP_LEGACY_SERVER_CONNECT option + self._ssl_context_noverify = ssl_context + return self._ssl_context_noverify @asynccontextmanager async def _acatch(self, url, raise_error): From c7aca036bacadbe1a781e1b4183688df848c9262 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 18:49:46 -0400 Subject: [PATCH 28/49] updated release history --- docs/release_history.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/release_history.md b/docs/release_history.md index 9bb4a8367..87a22eae9 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,3 +1,40 @@ +## v1.1.1 +October 11, 2023 + +Includes webhook output modules - Discord, Slack, and Teams! + +![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/72e3e940-a41a-4c7a-952e-49a1d7cae526) + +### Improvements +- https://github.com/blacklanternsecurity/bbot/pull/677 +- https://github.com/blacklanternsecurity/bbot/pull/674 +- https://github.com/blacklanternsecurity/bbot/pull/683 +- https://github.com/blacklanternsecurity/bbot/pull/740 +- https://github.com/blacklanternsecurity/bbot/pull/743 +- https://github.com/blacklanternsecurity/bbot/pull/748 +- https://github.com/blacklanternsecurity/bbot/pull/749 +- https://github.com/blacklanternsecurity/bbot/pull/751 +- https://github.com/blacklanternsecurity/bbot/pull/692 + +### Bugfixes +- https://github.com/blacklanternsecurity/bbot/pull/691 +- https://github.com/blacklanternsecurity/bbot/pull/684 +- https://github.com/blacklanternsecurity/bbot/pull/669 +- https://github.com/blacklanternsecurity/bbot/pull/664 +- https://github.com/blacklanternsecurity/bbot/pull/737 +- https://github.com/blacklanternsecurity/bbot/pull/741 +- https://github.com/blacklanternsecurity/bbot/pull/744 +- https://github.com/blacklanternsecurity/bbot/issues/760 +- https://github.com/blacklanternsecurity/bbot/issues/759 +- https://github.com/blacklanternsecurity/bbot/issues/758 +- https://github.com/blacklanternsecurity/bbot/pull/764 +- https://github.com/blacklanternsecurity/bbot/pull/773 + +### New Modules +- https://github.com/blacklanternsecurity/bbot/pull/689 +- https://github.com/blacklanternsecurity/bbot/pull/665 +- https://github.com/blacklanternsecurity/bbot/pull/663 + ## v1.1.0 August 4, 2023 From 6b1e320d1511ffc36e0c6e045abc7a319c0ebc77 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 19:30:29 -0400 Subject: [PATCH 29/49] fix test_modules_basic tests --- bbot/test/test_step_1/test_modules_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 77c25a7a1..47d39e1a5 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -82,6 +82,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h modules=list(set(available_modules + available_internal_modules)), output_modules=list(available_output_modules), config=bbot_config, + force_start=True, ) scan2.helpers.dns.fallback_nameservers_file = fallback_nameservers await scan2.load_modules() From f32448a43d1d2020fcf5ca3f006e29b120b98e0c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 17 Oct 2023 20:07:12 -0400 Subject: [PATCH 30/49] working on tests --- bbot/test/test_step_1/test_modules_basic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 47d39e1a5..b4d61f516 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -175,9 +175,9 @@ async def test_modules_basic_perhostonly(scan, helpers, events, bbot_config, bbo "evilcorp.com", modules=list(set(available_modules + available_internal_modules)), config=bbot_config, + force_start=True, ) - await per_host_scan.load_modules() await per_host_scan.setup_modules() per_host_scan.status = "RUNNING" @@ -215,6 +215,7 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, b "evilcorp.com", modules=list(set(available_modules + available_internal_modules)), config=bbot_config, + force_start=True, ) await per_domain_scan.load_modules() From 663cf444978585561f6ebfec5e0f55ef366f09e0 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 19 Oct 2023 11:27:47 -0400 Subject: [PATCH 31/49] emitting DNS_NAME instead --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 296cb5a59..95a5a848f 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -51,7 +51,7 @@ async def search(self, content, event, **kwargs): self.report(domain, event, **kwargs) def report(self, domain, event, **kwargs): - self.excavate.emit_event(domain, "DNS_NAME_UNRESOLVED", source=event) + self.excavate.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) class HostnameExtractor(BaseExtractor): From 896db82b50f6fb83935b6f70d2194c2f18ea9d42 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sun, 22 Oct 2023 00:22:38 -0400 Subject: [PATCH 32/49] added a bunch of azure domains --- bbot/core/helpers/cloud/azure.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/bbot/core/helpers/cloud/azure.py b/bbot/core/helpers/cloud/azure.py index 69f3b6dda..83ceb5b03 100644 --- a/bbot/core/helpers/cloud/azure.py +++ b/bbot/core/helpers/cloud/azure.py @@ -2,25 +2,47 @@ class Azure(BaseCloudProvider): + # mostly pulled from https://learn.microsoft.com/en-us/azure/azure-government/compare-azure-government-global-azure domains = [ + "azconfig.io", "azmk8s.io", "azure-api.net", + "azure-api.us", + "azure-automation.net", + "azure-automation.us", + "azure-devices.net", + "azure-devices.us", "azure-mobile.net", "azure.com", "azure.net", + "azure.us", "azurecontainer.io", "azurecr.io", + "azurecr.us", "azuredatalakestore.net", "azureedge.net", "azurefd.net", "azurehdinsight.net", + "azurehdinsight.us", "azurewebsites.net", + "botframework.com", "cloudapp.net", - "windows.net", + "loganalytics.io", + "loganalytics.us", + "microsoft.us", + "microsoftonline.com", + "microsoftonline.us", "onmicrosoft.com", + "powerbi.com", + "powerbigov.us", "trafficmanager.net", + "usgovcloudapi.net", + "usgovtrafficmanager.net", "visualstudio.com", "vo.msecnd.net", + "windows.net", + "windowsazure.com", + "windowsazure.us", ] bucket_name_regex = r"[a-z0-9][a-z0-9-_\.]{1,61}[a-z0-9]" From 9f52a1bdff2b4f7e758039b3ecd814f0ef80741a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 23 Oct 2023 11:24:15 -0400 Subject: [PATCH 33/49] allow override of batch_size / max_event_handlers --- bbot/core/configurator/__init__.py | 21 +++++++++++++++------ bbot/modules/badsecrets.py | 1 - bbot/modules/base.py | 24 +++++++++++++++++------- bbot/modules/deadly/nuclei.py | 3 +-- bbot/modules/dnscommonsrv.py | 2 +- bbot/modules/dnszonetransfer.py | 2 +- bbot/modules/fingerprintx.py | 4 ++-- bbot/modules/gowitness.py | 2 +- bbot/modules/httpx.py | 2 +- bbot/modules/iis_shortnames.py | 2 +- bbot/modules/nmap.py | 2 +- bbot/modules/nsec.py | 2 +- bbot/modules/oauth.py | 2 +- bbot/modules/output/neo4j.py | 2 +- bbot/modules/output/teams.py | 2 +- bbot/modules/paramminer_cookies.py | 2 +- bbot/modules/paramminer_headers.py | 2 +- bbot/modules/sslcert.py | 2 +- bbot/modules/subdomain_hijack.py | 2 +- bbot/modules/telerik.py | 2 +- bbot/modules/wappalyzer.py | 2 +- 21 files changed, 51 insertions(+), 34 deletions(-) diff --git a/bbot/core/configurator/__init__.py b/bbot/core/configurator/__init__.py index 58526317b..3d3a1fe60 100644 --- a/bbot/core/configurator/__init__.py +++ b/bbot/core/configurator/__init__.py @@ -1,4 +1,5 @@ import os +import re import sys from pathlib import Path from omegaconf import OmegaConf @@ -41,12 +42,20 @@ sentinel = object() +exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$") + + def check_cli_args(): - for c in args.cli_config: - if not is_file(c): - c = c.split("=")[0].strip() - v = OmegaConf.select(default_config, c, default=sentinel) - if v is sentinel: + conf = [a for a in args.cli_config if not is_file(a)] + all_options = None + for c in conf: + c = c.split("=")[0].strip() + v = OmegaConf.select(default_config, c, default=sentinel) + # if option isn't in the default config + if v is sentinel: + if exclude_from_validation.match(c): + continue + if all_options is None: from ...modules import module_loader modules_options = set() @@ -54,7 +63,7 @@ def check_cli_args(): modules_options.update(set(o[0] for o in module_options)) global_options = set(default_config.keys()) - {"modules", "output_modules"} all_options = global_options.union(modules_options) - match_and_exit(c, all_options, msg="module option") + match_and_exit(c, all_options, msg="module option") def ensure_config_files(): diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 3ca03c2bf..cc52feecb 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -10,7 +10,6 @@ class badsecrets(BaseModule): produced_events = ["FINDING", "VULNERABILITY"] flags = ["active", "safe", "web-basic", "web-thorough"] meta = {"description": "Library for detecting known or weak secrets across many web frameworks"} - max_event_handlers = 2 deps_pip = ["badsecrets~=0.4"] @property diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 9101c800a..2c3643411 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -94,8 +94,8 @@ class BaseModule: target_only = False in_scope_only = False - max_event_handlers = 1 - batch_size = 1 + _max_event_handlers = 1 + _batch_size = 1 batch_wait = 10 failed_request_abort_threshold = 5 @@ -288,6 +288,20 @@ async def ping(self): """ return + @property + def batch_size(self): + batch_size = self.options.get("batch_size", None) + if batch_size is None: + batch_size = self._batch_size + return batch_size + + @property + def max_event_handlers(self): + max_event_handlers = self.options.get("max_event_handlers", None) + if max_event_handlers is None: + max_event_handlers = self._max_event_handlers + return max_event_handlers + @property def auth_secret(self): """Indicates if the module is properly configured for authentication. @@ -484,12 +498,8 @@ def num_incoming_events(self): ret = self.incoming_event_queue.qsize() return ret - @property - def _max_event_handlers(self): - return self.max_event_handlers - def start(self): - self._tasks = [asyncio.create_task(self._worker()) for _ in range(self._max_event_handlers)] + self._tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)] async def _setup(self): """ diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 8f2feff5c..40d4614ea 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -10,8 +10,6 @@ class nuclei(BaseModule): flags = ["active", "aggressive"] meta = {"description": "Fast and customisable vulnerability scanner"} - batch_size = 25 - options = { "version": "2.9.15", "tags": "", @@ -49,6 +47,7 @@ class nuclei(BaseModule): ] deps_pip = ["pyyaml~=6.0"] in_scope_only = True + _batch_size = 25 async def setup(self): # attempt to update nuclei templates diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 6c80212d7..538f51621 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -94,7 +94,7 @@ class dnscommonsrv(BaseModule): produced_events = ["DNS_NAME"] flags = ["subdomain-enum", "passive", "safe"] meta = {"description": "Check for common SRV records"} - max_event_handlers = 5 + _max_event_handlers = 5 async def filter_event(self, event): # skip SRV wildcards diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index bb9819fbb..d950514b5 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -11,7 +11,7 @@ class dnszonetransfer(BaseModule): meta = {"description": "Attempt DNS zone transfers"} options = {"timeout": 10} options_desc = {"timeout": "Max seconds to wait before timing out"} - max_event_handlers = 5 + _max_event_handlers = 5 suppress_dupes = False async def setup(self): diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index 3bc36da66..be5695541 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -10,8 +10,8 @@ class fingerprintx(BaseModule): meta = {"description": "Fingerprint exposed services like RDP, SSH, MySQL, etc."} options = {"version": "1.1.4"} options_desc = {"version": "fingerprintx version"} - batch_size = 10 - max_event_handlers = 2 + _batch_size = 10 + _max_event_handlers = 2 _priority = 2 deps_ansible = [ diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index f19c5ed49..5f4c4a5e8 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -11,7 +11,6 @@ class gowitness(BaseModule): produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"] flags = ["active", "safe", "web-screenshots"] meta = {"description": "Take screenshots of webpages"} - batch_size = 100 options = { "version": "2.4.2", "threads": 4, @@ -76,6 +75,7 @@ class gowitness(BaseModule): }, }, ] + _batch_size = 100 # visit up to and including the scan's configured search distance plus one # this is one hop further than the default scope_distance_modifier = 1 diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index ef77668db..ff53e9972 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -10,7 +10,6 @@ class httpx(BaseModule): flags = ["active", "safe", "web-basic", "web-thorough", "social-enum", "subdomain-enum", "cloud-enum"] meta = {"description": "Visit webpages. Many other modules rely on httpx"} - batch_size = 500 options = {"threads": 50, "in_scope_only": True, "version": "1.2.5", "max_response_size": 5242880} options_desc = { "threads": "Number of httpx threads to use", @@ -31,6 +30,7 @@ class httpx(BaseModule): ] scope_distance_modifier = 1 + _batch_size = 500 _priority = 2 async def setup(self): diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 9e3566ac8..f9914e316 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -21,7 +21,7 @@ class iis_shortnames(BaseModule): } in_scope_only = True - max_event_handlers = 8 + _max_event_handlers = 8 async def detect(self, target): technique = None diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py index e2900ab1b..5e485e5ac 100644 --- a/bbot/modules/nmap.py +++ b/bbot/modules/nmap.py @@ -19,7 +19,7 @@ class nmap(BaseModule): "timing": "-T<0-5>: Set timing template (higher is faster)", "skip_host_discovery": "skip host discovery (-Pn)", } - max_event_handlers = 2 + _max_event_handlers = 2 batch_size = 256 _priority = 2 diff --git a/bbot/modules/nsec.py b/bbot/modules/nsec.py index 2ab254da7..cafed732e 100644 --- a/bbot/modules/nsec.py +++ b/bbot/modules/nsec.py @@ -6,7 +6,7 @@ class NSEC(BaseModule): produced_events = ["DNS_NAME"] flags = ["subdomain-enum", "passive", "safe"] meta = {"description": "Enumerate subdomains by NSEC-walking"} - max_event_handlers = 5 + _max_event_handlers = 5 async def filter_event(self, event): if "ns-record" in event.tags: diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index 0bf6457c2..f4d592511 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -13,7 +13,7 @@ class OAUTH(BaseModule): in_scope_only = False scope_distance_modifier = 1 - max_event_handlers = 2 + _max_event_handlers = 2 async def setup(self): self.processed = set() diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 18bda8bad..2fc05fc9c 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -16,7 +16,7 @@ class neo4j(BaseOutputModule): "password": "Neo4j password", } deps_pip = ["git+https://github.com/blacklanternsecurity/py2neo"] - batch_size = 50 + _batch_size = 50 async def setup(self): try: diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 0953fef5f..f858d897d 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -10,7 +10,7 @@ class Teams(Discord): "event_types": "Types of events to send", "min_severity": "Only allow VULNERABILITY events of this severity or highter", } - max_event_handlers = 5 + _max_event_handlers = 5 good_status_code = 200 content_key = "text" diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index 12b9ac32b..251931f58 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -25,7 +25,7 @@ class paramminer_cookies(paramminer_headers): options_desc = {"wordlist": "Define the wordlist to be used to derive cookies"} scanned_hosts = [] boring_words = set() - max_event_handlers = 12 + _max_event_handlers = 12 in_scope_only = True compare_mode = "cookie" default_wordlist = "paramminer_parameters.txt" diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 8632ddf0d..65880d9c8 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -69,7 +69,7 @@ class paramminer_headers(BaseModule): "zx-request-id", "zx-timer", } - max_event_handlers = 12 + _max_event_handlers = 12 in_scope_only = True compare_mode = "header" default_wordlist = "paramminer_headers.txt" diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index de598dd73..01dc9844b 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -18,7 +18,7 @@ class sslcert(BaseModule): options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"} deps_apt = ["openssl"] deps_pip = ["pyOpenSSL~=23.1.1"] - max_event_handlers = 25 + _max_event_handlers = 25 scope_distance_modifier = 1 _priority = 2 diff --git a/bbot/modules/subdomain_hijack.py b/bbot/modules/subdomain_hijack.py index 201e8b4a3..a25b1d64b 100644 --- a/bbot/modules/subdomain_hijack.py +++ b/bbot/modules/subdomain_hijack.py @@ -16,7 +16,7 @@ class subdomain_hijack(BaseModule): } options_desc = {"fingerprints": "URL or path to fingerprints.json"} scope_distance_modifier = 3 - max_event_handlers = 5 + _max_event_handlers = 5 async def setup(self): fingerprints_url = self.config.get("fingerprints") diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index f65ad9df2..e012b6af6 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -157,7 +157,7 @@ class telerik(BaseModule): }, ] - max_event_handlers = 5 + _max_event_handlers = 5 async def setup(self): self.timeout = self.scan.config.get("httpx_timeout", 5) diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index a372d1791..7bc10b448 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -20,7 +20,7 @@ class wappalyzer(BaseModule): deps_pip = ["python-Wappalyzer~=0.3.1"] # accept all events regardless of scope distance scope_distance_modifier = None - max_event_handlers = 5 + _max_event_handlers = 5 async def setup(self): self.wappalyzer = await self.scan.run_in_executor(Wappalyzer.latest) From 8c5f3e594803b77491f534c7ce374fcdee349c81 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 23 Oct 2023 11:30:35 -0400 Subject: [PATCH 34/49] allow override of batch_size / max_event_handlers --- bbot/modules/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 2c3643411..0c967f400 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -291,7 +291,9 @@ async def ping(self): @property def batch_size(self): batch_size = self.options.get("batch_size", None) - if batch_size is None: + # only allow overriding the batch size if its default value is greater than 1 + # this prevents modules from being accidentally neutered by an incorect batch_size setting + if batch_size is None or (self._batch_size > 1 and batch_size <= 1): batch_size = self._batch_size return batch_size From 11bdbe8107a2402bb71cd4610ac57d54beeef9dc Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 23 Oct 2023 12:00:57 -0400 Subject: [PATCH 35/49] fix bug --- bbot/modules/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 0c967f400..125b9c14b 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -290,16 +290,16 @@ async def ping(self): @property def batch_size(self): - batch_size = self.options.get("batch_size", None) + batch_size = self.config.get("batch_size", None) # only allow overriding the batch size if its default value is greater than 1 # this prevents modules from being accidentally neutered by an incorect batch_size setting - if batch_size is None or (self._batch_size > 1 and batch_size <= 1): + if batch_size is None or self._batch_size == 1: batch_size = self._batch_size return batch_size @property def max_event_handlers(self): - max_event_handlers = self.options.get("max_event_handlers", None) + max_event_handlers = self.config.get("max_event_handlers", None) if max_event_handlers is None: max_event_handlers = self._max_event_handlers return max_event_handlers From 5c16e5f82f775ce6d69da0822aae3714ffa5b2f4 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 23 Oct 2023 12:01:05 -0400 Subject: [PATCH 36/49] Adding nuclei retries / batchsize option, test rework --- bbot/modules/deadly/nuclei.py | 12 +++++-- .../module_tests/test_module_nuclei.py | 35 +++++++++++++++++-- docs/modules/nuclei.md | 2 ++ 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 8f2feff5c..728221100 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -10,8 +10,6 @@ class nuclei(BaseModule): flags = ["active", "aggressive"] meta = {"description": "Fast and customisable vulnerability scanner"} - batch_size = 25 - options = { "version": "2.9.15", "tags": "", @@ -23,6 +21,8 @@ class nuclei(BaseModule): "etags": "", "budget": 1, "directory_only": True, + "retries": 0, + "batch_size": 200, } options_desc = { "version": "nuclei version", @@ -35,6 +35,8 @@ class nuclei(BaseModule): "etags": "tags to exclude from the scan", "budget": "Used in budget mode to set the number of requests which will be alloted to the nuclei scan", "directory_only": "Filter out 'file' URL event (default True)", + "retries": "number of times to retry a failed request (default 0)", + "batch_size": "Number of targets to send to Nuclei per batch (default 200)", } deps_ansible = [ { @@ -51,6 +53,9 @@ class nuclei(BaseModule): in_scope_only = True async def setup(self): + # setup batch size + self.batch_size = int(self.config.get("batch_size", 200)) + # attempt to update nuclei templates self.nuclei_templates_dir = self.helpers.tools_dir / "nuclei-templates" self.info("Updating Nuclei templates") @@ -85,6 +90,7 @@ async def setup(self): self.info(f"Limiting nuclei templates to the following severites: [{self.severity}]") self.iserver = self.scan.config.get("interactsh_server", None) self.itoken = self.scan.config.get("interactsh_token", None) + self.retries = int(self.config.get("retries", 0)) if self.mode not in ("technology", "severe", "manual", "budget"): self.warning(f"Unable to initialize nuclei: invalid mode selected: [{self.mode}]") @@ -185,6 +191,8 @@ async def execute_nuclei(self, nuclei_input): self.concurrency, "-disable-update-check", "-stats-json", + "-retries", + self.retries, ] if self.helpers.system_resolvers: diff --git a/bbot/test/test_step_2/module_tests/test_module_nuclei.py b/bbot/test/test_step_2/module_tests/test_module_nuclei.py index cd27ba3b6..f063cf006 100644 --- a/bbot/test/test_step_2/module_tests/test_module_nuclei.py +++ b/bbot/test/test_step_2/module_tests/test_module_nuclei.py @@ -89,7 +89,6 @@ class TestNucleiTechnology(TestNucleiManual): } async def setup_before_prep(self, module_test): - self.caplog = module_test.request_fixture.getfixturevalue("caplog") expect_args = {"method": "GET", "uri": "/"} respond_args = { "response_data": "", @@ -98,10 +97,11 @@ async def setup_before_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - if "Using Interactsh Server" in self.caplog.text: - return False assert any(e.type == "FINDING" and "apache" in e.data["description"] for e in events) + with open(module_test.scan.home / "debug.log") as f: + assert "Using Interactsh Server" not in f.read() + class TestNucleiBudget(TestNucleiManual): config_overrides = { @@ -123,3 +123,32 @@ async def setup_before_prep(self, module_test): def check(self, module_test, events): assert any(e.type == "FINDING" and "SpiderFoot" in e.data["description"] for e in events) + + +class TestNucleiRetries(TestNucleiManual): + config_overrides = { + "interactsh_disable": True, + "modules": {"nuclei": {"tags": "musictraveler"}}, + } + + async def setup_before_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = { + "response_data": "content", + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + with open(module_test.scan.home / "debug.log") as f: + assert "-retries 0" in f.read() + + +class TestNucleiRetriesCustom(TestNucleiRetries): + config_overrides = { + "interactsh_disable": True, + "modules": {"nuclei": {"tags": "musictraveler", "retries": 1}}, + } + + def check(self, module_test, events): + with open(module_test.scan.home / "debug.log") as f: + assert "-retries 1" in f.read() diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md index 6beb3f870..e7c6d4247 100644 --- a/docs/modules/nuclei.md +++ b/docs/modules/nuclei.md @@ -32,6 +32,8 @@ The Nuclei module has many configuration options: | etags | Tags to exclude from the scan | | | directory_only | When on, limits scan to only "directory" URLs (omit endpoints) | True | | budget | Used in budget mode to set the number of requests which will be allotted | 1 | +| retries | Mumber of times to retry a failed request | 0 | +| batch_size | The number of targets BBOT will pass to Nuclei at a time | 200 | Most of these you probably will **NOT** want to change. In particular, we strongly advise against changing the version of Nuclei, as it's very likely the latest version won't work right with BBOT. From 8de834322b12acccccf5e5be89b0d5d3d7cb4608 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 23 Oct 2023 17:13:18 -0400 Subject: [PATCH 37/49] removing unnecessary setup --- bbot/modules/deadly/nuclei.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 6600b7b16..94d664b6f 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -54,9 +54,6 @@ class nuclei(BaseModule): _batch_size = 25 async def setup(self): - # setup batch size - self.batch_size = int(self.config.get("batch_size", 200)) - # attempt to update nuclei templates self.nuclei_templates_dir = self.helpers.tools_dir / "nuclei-templates" self.info("Updating Nuclei templates") From 43aa8b481230a6790672a440386d2f96a38d0b36 Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:21:41 -0600 Subject: [PATCH 38/49] Update release_history.md --- docs/release_history.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/release_history.md b/docs/release_history.md index 87a22eae9..50c344708 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,3 +1,24 @@ +## v1.1.2 +October 24, 2023 + +### Improvements +- https://github.com/blacklanternsecurity/bbot/pull/776 +- https://github.com/blacklanternsecurity/bbot/pull/772 +- https://github.com/blacklanternsecurity/bbot/pull/783 +- https://github.com/blacklanternsecurity/bbot/pull/790 +- https://github.com/blacklanternsecurity/bbot/pull/797 +- https://github.com/blacklanternsecurity/bbot/pull/798 +- https://github.com/blacklanternsecurity/bbot/pull/799 + +### Bugfixes +- https://github.com/blacklanternsecurity/bbot/pull/780 +- https://github.com/blacklanternsecurity/bbot/pull/787 +- https://github.com/blacklanternsecurity/bbot/pull/788 +- https://github.com/blacklanternsecurity/bbot/pull/791 + +### New Modules +- https://github.com/blacklanternsecurity/bbot/pull/774 + ## v1.1.1 October 11, 2023 From 67db834aaf2ade4a2bae1804beefe1dc75d42f67 Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Mon, 23 Oct 2023 22:43:24 +0000 Subject: [PATCH 39/49] Refresh module docs --- docs/scanning/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 90282ef66..fb9d5544c 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -250,12 +250,14 @@ Many modules accept their own configuration options. These options have the abil | modules.nmap.timing | str |` -T<0-5>: Set timing template (higher is faster) `| T4 | | modules.nmap.top_ports | int | top ports to scan | 100 | | modules.ntlm.try_all | bool | Try every NTLM endpoint | False | +| modules.nuclei.batch_size | int | Number of targets to send to Nuclei per batch (default 200) | 200 | | modules.nuclei.budget | int | Used in budget mode to set the number of requests which will be alloted to the nuclei scan | 1 | | modules.nuclei.concurrency | int | maximum number of templates to be executed in parallel (default 25) | 25 | | modules.nuclei.directory_only | bool | Filter out 'file' URL event (default True) | True | | modules.nuclei.etags | str | tags to exclude from the scan | | | modules.nuclei.mode | str | manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual | | modules.nuclei.ratelimit | int | maximum number of requests to send per second (default 150) | 150 | +| modules.nuclei.retries | int | number of times to retry a failed request (default 0) | 0 | | modules.nuclei.severity | str | Filter based on severity field available in the template. | | | modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | | modules.nuclei.templates | str | template or template directory paths to include in the scan | | From 0c699e10bef146ee0c21eaef8752f22f4e7b42d9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 23 Oct 2023 22:20:37 -0400 Subject: [PATCH 40/49] update docs --- README.md | 8 ++++++-- docs/dev/index.md | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2c8ff90f5..203e4c394 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,9 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/scann ## BBOT as a Python library -BBOT exposes a Python API that allows it to be used for all kinds of fun and nefarious purposes, like a [Discord Bot that responds to `/scan evilcorp.com`](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example). +BBOT exposes a Python API that allows it to be used for all kinds of fun and nefarious purposes, like a [Discord Bot](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example) that responds to the `/scan` command. + +![![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9)](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example) **Synchronous** @@ -167,7 +169,9 @@ asyncio.run(main()) - [Troubleshooting](https://www.blacklanternsecurity.com/bbot/troubleshooting) -## Acknowledgements +## Contribution + +Collaboration is what makes BBOT great. We welcome contributions - and not just in the form of code, but ideas, too! If you have an idea for a new feature, please let us know in [Discussions](https://github.com/blacklanternsecurity/bbot/discussions). If you want to get your hands dirty, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). There you can find setup instructions and a simple tutorial on how to write a BBOT module. We also have extensive [Developer Documentation](https://www.blacklanternsecurity.com/bbot/dev/). Thanks to these amazing people for contributing to BBOT! :heart: diff --git a/docs/dev/index.md b/docs/dev/index.md index 5a9bf88d0..093a3aefb 100644 --- a/docs/dev/index.md +++ b/docs/dev/index.md @@ -6,6 +6,8 @@ Documented in this section are commonly-used classes and functions within BBOT, ## Discord Bot Example +![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) + Below is a simple Discord bot designed to run BBOT scans. ```python From 560c899c27c6728f368a172a5ffc460ccafb2391 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 23 Oct 2023 22:26:39 -0400 Subject: [PATCH 41/49] fix readme gif --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 203e4c394..4e040ae42 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/scann BBOT exposes a Python API that allows it to be used for all kinds of fun and nefarious purposes, like a [Discord Bot](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example) that responds to the `/scan` command. -![![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9)](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example) +![bbot-discord](https://github.com/blacklanternsecurity/bbot/assets/20261699/22b268a2-0dfd-4c2a-b7c5-548c0f2cc6f9) **Synchronous** From ffc4a188467e4fa4874fd54c5985821543403887 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 23 Oct 2023 22:31:09 -0400 Subject: [PATCH 42/49] update README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4e040ae42..5d781e6aa 100644 --- a/README.md +++ b/README.md @@ -171,11 +171,11 @@ asyncio.run(main()) ## Contribution -Collaboration is what makes BBOT great. We welcome contributions - and not just in the form of code, but ideas, too! If you have an idea for a new feature, please let us know in [Discussions](https://github.com/blacklanternsecurity/bbot/discussions). If you want to get your hands dirty, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). There you can find setup instructions and a simple tutorial on how to write a BBOT module. We also have extensive [Developer Documentation](https://www.blacklanternsecurity.com/bbot/dev/). +BBOT is constantly being improved by the community. Every day it grows more powerful! -Thanks to these amazing people for contributing to BBOT! :heart: +We welcome contributions. Not just code, but ideas too! If you have an idea for a new feature, please let us know in [Discussions](https://github.com/blacklanternsecurity/bbot/discussions). If you want to get your hands dirty, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). There you can find setup instructions and a simple tutorial on how to write a BBOT module. We also have extensive [Developer Documentation](https://www.blacklanternsecurity.com/bbot/dev/). -If you're interested in contributing to BBOT, or just curious how it works under the hood, see [Contribution](https://www.blacklanternsecurity.com/bbot/contribution/). +Thanks to these amazing people for contributing to BBOT! :heart:

From 2e8658845efedbd50e0896282f218439f3b67d48 Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:12:32 -0400 Subject: [PATCH 43/49] Update release_history.md --- docs/release_history.md | 311 ++++++++++++++++++++-------------------- 1 file changed, 156 insertions(+), 155 deletions(-) diff --git a/docs/release_history.md b/docs/release_history.md index 50c344708..54ae551e2 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,14 +1,15 @@ -## v1.1.2 -October 24, 2023 - +## v1.1.2 +October 24, 2023 + ### Improvements - https://github.com/blacklanternsecurity/bbot/pull/776 - https://github.com/blacklanternsecurity/bbot/pull/772 - https://github.com/blacklanternsecurity/bbot/pull/783 - https://github.com/blacklanternsecurity/bbot/pull/790 - https://github.com/blacklanternsecurity/bbot/pull/797 -- https://github.com/blacklanternsecurity/bbot/pull/798 +- https://github.com/blacklanternsecurity/bbot/pull/798 - https://github.com/blacklanternsecurity/bbot/pull/799 +- https://github.com/blacklanternsecurity/bbot/pull/801 ### Bugfixes - https://github.com/blacklanternsecurity/bbot/pull/780 @@ -17,154 +18,154 @@ October 24, 2023 - https://github.com/blacklanternsecurity/bbot/pull/791 ### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/774 - -## v1.1.1 -October 11, 2023 - -Includes webhook output modules - Discord, Slack, and Teams! - -![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/72e3e940-a41a-4c7a-952e-49a1d7cae526) - -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/677 -- https://github.com/blacklanternsecurity/bbot/pull/674 -- https://github.com/blacklanternsecurity/bbot/pull/683 -- https://github.com/blacklanternsecurity/bbot/pull/740 -- https://github.com/blacklanternsecurity/bbot/pull/743 -- https://github.com/blacklanternsecurity/bbot/pull/748 -- https://github.com/blacklanternsecurity/bbot/pull/749 -- https://github.com/blacklanternsecurity/bbot/pull/751 -- https://github.com/blacklanternsecurity/bbot/pull/692 - -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/691 -- https://github.com/blacklanternsecurity/bbot/pull/684 -- https://github.com/blacklanternsecurity/bbot/pull/669 -- https://github.com/blacklanternsecurity/bbot/pull/664 -- https://github.com/blacklanternsecurity/bbot/pull/737 -- https://github.com/blacklanternsecurity/bbot/pull/741 -- https://github.com/blacklanternsecurity/bbot/pull/744 -- https://github.com/blacklanternsecurity/bbot/issues/760 -- https://github.com/blacklanternsecurity/bbot/issues/759 -- https://github.com/blacklanternsecurity/bbot/issues/758 -- https://github.com/blacklanternsecurity/bbot/pull/764 -- https://github.com/blacklanternsecurity/bbot/pull/773 - -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/689 -- https://github.com/blacklanternsecurity/bbot/pull/665 -- https://github.com/blacklanternsecurity/bbot/pull/663 - -## v1.1.0 -August 4, 2023 - -**New Features**: - -- Complete Asyncification -- Documentation + auto-updating pipelines -- Ability to list flags and their descriptions with `-lf` -- Fine-grained rate-limiting for HTTP and DNS - -**Improvements / Fixes**: - -- Better tests (one for each individual module, 91% test coverage) -- New and improved paramminer modules -- Misc bugfixes - -**New Modules**: - -- Git (detects exposed .git folder on websites) -- [Subdomain Center](https://www.subdomain.center/) (subdomain enumeration) -- [Columbus API](https://columbus.elmasy.com/) (subdomain enumeration) -- MySSL (subdomain enumeration) -- Sitedossier (subdomain enumeration) -- Digitorus (subdomain enumeration) -- Nmap (port scanner, more reliable than naabu) - - Naabu has been removed due to reliability issues -- NSEC (DNSSEC zone-walking for subdomain enumeration) -- OAUTH (Enumerates OAUTH / OpenID-Connect, detects sprayable endpoints) -- Azure Realm (Detects Managed/Federated Azure Tenants) -- Subdomains output module - - -## v1.0.5 -March 10, 2023 - -**New Modules**: - -- [Badsecrets](https://github.com/blacklanternsecurity/badsecrets) ([blacklist3r](https://github.com/NotSoSecure/Blacklist3r) but better!) -- Subdomain Hijacking (uses [can-i-take-over-xyz](https://github.com/EdOverflow/can-i-take-over-xyz)) -- [WafW00f](https://github.com/EnableSecurity/wafw00f) -- [Fingerprintx](https://github.com/praetorian-inc/fingerprintx) -- [Masscan](https://github.com/robertdavidgraham/masscan) -- Robots.txt -- Web Report -- IIS shortnames (Pure Python rewrite) - -**New Features**: - -- Automatic tagging of cloud resources (with [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck)) -- Significant performance increases -- Bug fixes -- Better tests + code coverage -- Support for punycode (non-ascii) domains -- Better support for non-64-bit systems -- Enter key now toggles verbosity during scan - - -## v1.0.4 -December 15, 2022 - -**New Modules**: - -- Storage buckets: - - Azure - - GCP - - AWS - - DigitalOcean -- ipstack (geolocation) -- BeVigil -- ASN (rewrite) - -**New Features**: - -- Colored vulnerabilities on CLI -- Log full nuclei output -- Various bugfixes -- Better handling of: - - DNS wildcards - - Infinite DNS-record chains - - Infinite HTTP redirects -- Improved module tests - -## v1.0.3 -October 12, 2022 - -**Changes**: - -- Tag URL events with their corresponding IP address -- Automatic docker hub publishing -- Added `retries` option for httpx module -- Added `asset_inventory` output module -- Improvements to nuclei module -- Avoid unnecessary failed sudo attempts during dependency install -- Improved Python API -- Added AnubisDB module -- Various bugfixes -- Add examples to `--help` output -- Reduce annoying warnings on free API modules -- Update iis_shortnames .jar dependency -- Updated documentation to explain targets, whitelists, blacklists -- Added help for module-specific options -- Added warning if unable to validate public DNS servers (for massdns) -- Various performance optimizations -- Various bugfixes -- Fix Pypi auto-publishing -- Added bug report template -- Added examples in README -- Improved wildcard detection -- Added DNS retry functionality -- Improved excavate hostname extraction -- Added command-line option for installing all dependencies -- Improved gowitness dependency install, improved tests +- https://github.com/blacklanternsecurity/bbot/pull/774 + +## v1.1.1 +October 11, 2023 + +Includes webhook output modules - Discord, Slack, and Teams! + +![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/72e3e940-a41a-4c7a-952e-49a1d7cae526) + +### Improvements +- https://github.com/blacklanternsecurity/bbot/pull/677 +- https://github.com/blacklanternsecurity/bbot/pull/674 +- https://github.com/blacklanternsecurity/bbot/pull/683 +- https://github.com/blacklanternsecurity/bbot/pull/740 +- https://github.com/blacklanternsecurity/bbot/pull/743 +- https://github.com/blacklanternsecurity/bbot/pull/748 +- https://github.com/blacklanternsecurity/bbot/pull/749 +- https://github.com/blacklanternsecurity/bbot/pull/751 +- https://github.com/blacklanternsecurity/bbot/pull/692 + +### Bugfixes +- https://github.com/blacklanternsecurity/bbot/pull/691 +- https://github.com/blacklanternsecurity/bbot/pull/684 +- https://github.com/blacklanternsecurity/bbot/pull/669 +- https://github.com/blacklanternsecurity/bbot/pull/664 +- https://github.com/blacklanternsecurity/bbot/pull/737 +- https://github.com/blacklanternsecurity/bbot/pull/741 +- https://github.com/blacklanternsecurity/bbot/pull/744 +- https://github.com/blacklanternsecurity/bbot/issues/760 +- https://github.com/blacklanternsecurity/bbot/issues/759 +- https://github.com/blacklanternsecurity/bbot/issues/758 +- https://github.com/blacklanternsecurity/bbot/pull/764 +- https://github.com/blacklanternsecurity/bbot/pull/773 + +### New Modules +- https://github.com/blacklanternsecurity/bbot/pull/689 +- https://github.com/blacklanternsecurity/bbot/pull/665 +- https://github.com/blacklanternsecurity/bbot/pull/663 + +## v1.1.0 +August 4, 2023 + +**New Features**: + +- Complete Asyncification +- Documentation + auto-updating pipelines +- Ability to list flags and their descriptions with `-lf` +- Fine-grained rate-limiting for HTTP and DNS + +**Improvements / Fixes**: + +- Better tests (one for each individual module, 91% test coverage) +- New and improved paramminer modules +- Misc bugfixes + +**New Modules**: + +- Git (detects exposed .git folder on websites) +- [Subdomain Center](https://www.subdomain.center/) (subdomain enumeration) +- [Columbus API](https://columbus.elmasy.com/) (subdomain enumeration) +- MySSL (subdomain enumeration) +- Sitedossier (subdomain enumeration) +- Digitorus (subdomain enumeration) +- Nmap (port scanner, more reliable than naabu) + - Naabu has been removed due to reliability issues +- NSEC (DNSSEC zone-walking for subdomain enumeration) +- OAUTH (Enumerates OAUTH / OpenID-Connect, detects sprayable endpoints) +- Azure Realm (Detects Managed/Federated Azure Tenants) +- Subdomains output module + + +## v1.0.5 +March 10, 2023 + +**New Modules**: + +- [Badsecrets](https://github.com/blacklanternsecurity/badsecrets) ([blacklist3r](https://github.com/NotSoSecure/Blacklist3r) but better!) +- Subdomain Hijacking (uses [can-i-take-over-xyz](https://github.com/EdOverflow/can-i-take-over-xyz)) +- [WafW00f](https://github.com/EnableSecurity/wafw00f) +- [Fingerprintx](https://github.com/praetorian-inc/fingerprintx) +- [Masscan](https://github.com/robertdavidgraham/masscan) +- Robots.txt +- Web Report +- IIS shortnames (Pure Python rewrite) + +**New Features**: + +- Automatic tagging of cloud resources (with [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck)) +- Significant performance increases +- Bug fixes +- Better tests + code coverage +- Support for punycode (non-ascii) domains +- Better support for non-64-bit systems +- Enter key now toggles verbosity during scan + + +## v1.0.4 +December 15, 2022 + +**New Modules**: + +- Storage buckets: + - Azure + - GCP + - AWS + - DigitalOcean +- ipstack (geolocation) +- BeVigil +- ASN (rewrite) + +**New Features**: + +- Colored vulnerabilities on CLI +- Log full nuclei output +- Various bugfixes +- Better handling of: + - DNS wildcards + - Infinite DNS-record chains + - Infinite HTTP redirects +- Improved module tests + +## v1.0.3 +October 12, 2022 + +**Changes**: + +- Tag URL events with their corresponding IP address +- Automatic docker hub publishing +- Added `retries` option for httpx module +- Added `asset_inventory` output module +- Improvements to nuclei module +- Avoid unnecessary failed sudo attempts during dependency install +- Improved Python API +- Added AnubisDB module +- Various bugfixes +- Add examples to `--help` output +- Reduce annoying warnings on free API modules +- Update iis_shortnames .jar dependency +- Updated documentation to explain targets, whitelists, blacklists +- Added help for module-specific options +- Added warning if unable to validate public DNS servers (for massdns) +- Various performance optimizations +- Various bugfixes +- Fix Pypi auto-publishing +- Added bug report template +- Added examples in README +- Improved wildcard detection +- Added DNS retry functionality +- Improved excavate hostname extraction +- Added command-line option for installing all dependencies +- Improved gowitness dependency install, improved tests From 3438beb37eb11b3aaf30725173b11e2bcc00a42d Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:20:27 -0400 Subject: [PATCH 44/49] Update release_history.md --- docs/release_history.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release_history.md b/docs/release_history.md index 54ae551e2..06fa8c397 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -3,7 +3,6 @@ October 24, 2023 ### Improvements - https://github.com/blacklanternsecurity/bbot/pull/776 -- https://github.com/blacklanternsecurity/bbot/pull/772 - https://github.com/blacklanternsecurity/bbot/pull/783 - https://github.com/blacklanternsecurity/bbot/pull/790 - https://github.com/blacklanternsecurity/bbot/pull/797 From cb7684657e35af622f2a1a277296c72df2151aeb Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Tue, 24 Oct 2023 18:21:40 +0000 Subject: [PATCH 45/49] Refresh module docs --- docs/release_history.md | 340 ++++++++++++++++++++-------------------- 1 file changed, 170 insertions(+), 170 deletions(-) diff --git a/docs/release_history.md b/docs/release_history.md index 06fa8c397..6281bc93e 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,170 +1,170 @@ -## v1.1.2 -October 24, 2023 - -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/776 -- https://github.com/blacklanternsecurity/bbot/pull/783 -- https://github.com/blacklanternsecurity/bbot/pull/790 -- https://github.com/blacklanternsecurity/bbot/pull/797 -- https://github.com/blacklanternsecurity/bbot/pull/798 -- https://github.com/blacklanternsecurity/bbot/pull/799 -- https://github.com/blacklanternsecurity/bbot/pull/801 - -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/780 -- https://github.com/blacklanternsecurity/bbot/pull/787 -- https://github.com/blacklanternsecurity/bbot/pull/788 -- https://github.com/blacklanternsecurity/bbot/pull/791 - -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/774 - -## v1.1.1 -October 11, 2023 - -Includes webhook output modules - Discord, Slack, and Teams! - -![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/72e3e940-a41a-4c7a-952e-49a1d7cae526) - -### Improvements -- https://github.com/blacklanternsecurity/bbot/pull/677 -- https://github.com/blacklanternsecurity/bbot/pull/674 -- https://github.com/blacklanternsecurity/bbot/pull/683 -- https://github.com/blacklanternsecurity/bbot/pull/740 -- https://github.com/blacklanternsecurity/bbot/pull/743 -- https://github.com/blacklanternsecurity/bbot/pull/748 -- https://github.com/blacklanternsecurity/bbot/pull/749 -- https://github.com/blacklanternsecurity/bbot/pull/751 -- https://github.com/blacklanternsecurity/bbot/pull/692 - -### Bugfixes -- https://github.com/blacklanternsecurity/bbot/pull/691 -- https://github.com/blacklanternsecurity/bbot/pull/684 -- https://github.com/blacklanternsecurity/bbot/pull/669 -- https://github.com/blacklanternsecurity/bbot/pull/664 -- https://github.com/blacklanternsecurity/bbot/pull/737 -- https://github.com/blacklanternsecurity/bbot/pull/741 -- https://github.com/blacklanternsecurity/bbot/pull/744 -- https://github.com/blacklanternsecurity/bbot/issues/760 -- https://github.com/blacklanternsecurity/bbot/issues/759 -- https://github.com/blacklanternsecurity/bbot/issues/758 -- https://github.com/blacklanternsecurity/bbot/pull/764 -- https://github.com/blacklanternsecurity/bbot/pull/773 - -### New Modules -- https://github.com/blacklanternsecurity/bbot/pull/689 -- https://github.com/blacklanternsecurity/bbot/pull/665 -- https://github.com/blacklanternsecurity/bbot/pull/663 - -## v1.1.0 -August 4, 2023 - -**New Features**: - -- Complete Asyncification -- Documentation + auto-updating pipelines -- Ability to list flags and their descriptions with `-lf` -- Fine-grained rate-limiting for HTTP and DNS - -**Improvements / Fixes**: - -- Better tests (one for each individual module, 91% test coverage) -- New and improved paramminer modules -- Misc bugfixes - -**New Modules**: - -- Git (detects exposed .git folder on websites) -- [Subdomain Center](https://www.subdomain.center/) (subdomain enumeration) -- [Columbus API](https://columbus.elmasy.com/) (subdomain enumeration) -- MySSL (subdomain enumeration) -- Sitedossier (subdomain enumeration) -- Digitorus (subdomain enumeration) -- Nmap (port scanner, more reliable than naabu) - - Naabu has been removed due to reliability issues -- NSEC (DNSSEC zone-walking for subdomain enumeration) -- OAUTH (Enumerates OAUTH / OpenID-Connect, detects sprayable endpoints) -- Azure Realm (Detects Managed/Federated Azure Tenants) -- Subdomains output module - - -## v1.0.5 -March 10, 2023 - -**New Modules**: - -- [Badsecrets](https://github.com/blacklanternsecurity/badsecrets) ([blacklist3r](https://github.com/NotSoSecure/Blacklist3r) but better!) -- Subdomain Hijacking (uses [can-i-take-over-xyz](https://github.com/EdOverflow/can-i-take-over-xyz)) -- [WafW00f](https://github.com/EnableSecurity/wafw00f) -- [Fingerprintx](https://github.com/praetorian-inc/fingerprintx) -- [Masscan](https://github.com/robertdavidgraham/masscan) -- Robots.txt -- Web Report -- IIS shortnames (Pure Python rewrite) - -**New Features**: - -- Automatic tagging of cloud resources (with [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck)) -- Significant performance increases -- Bug fixes -- Better tests + code coverage -- Support for punycode (non-ascii) domains -- Better support for non-64-bit systems -- Enter key now toggles verbosity during scan - - -## v1.0.4 -December 15, 2022 - -**New Modules**: - -- Storage buckets: - - Azure - - GCP - - AWS - - DigitalOcean -- ipstack (geolocation) -- BeVigil -- ASN (rewrite) - -**New Features**: - -- Colored vulnerabilities on CLI -- Log full nuclei output -- Various bugfixes -- Better handling of: - - DNS wildcards - - Infinite DNS-record chains - - Infinite HTTP redirects -- Improved module tests - -## v1.0.3 -October 12, 2022 - -**Changes**: - -- Tag URL events with their corresponding IP address -- Automatic docker hub publishing -- Added `retries` option for httpx module -- Added `asset_inventory` output module -- Improvements to nuclei module -- Avoid unnecessary failed sudo attempts during dependency install -- Improved Python API -- Added AnubisDB module -- Various bugfixes -- Add examples to `--help` output -- Reduce annoying warnings on free API modules -- Update iis_shortnames .jar dependency -- Updated documentation to explain targets, whitelists, blacklists -- Added help for module-specific options -- Added warning if unable to validate public DNS servers (for massdns) -- Various performance optimizations -- Various bugfixes -- Fix Pypi auto-publishing -- Added bug report template -- Added examples in README -- Improved wildcard detection -- Added DNS retry functionality -- Improved excavate hostname extraction -- Added command-line option for installing all dependencies -- Improved gowitness dependency install, improved tests +## v1.1.2 +October 24, 2023 + +### Improvements +- https://github.com/blacklanternsecurity/bbot/pull/776 +- https://github.com/blacklanternsecurity/bbot/pull/783 +- https://github.com/blacklanternsecurity/bbot/pull/790 +- https://github.com/blacklanternsecurity/bbot/pull/797 +- https://github.com/blacklanternsecurity/bbot/pull/798 +- https://github.com/blacklanternsecurity/bbot/pull/799 +- https://github.com/blacklanternsecurity/bbot/pull/801 + +### Bugfixes +- https://github.com/blacklanternsecurity/bbot/pull/780 +- https://github.com/blacklanternsecurity/bbot/pull/787 +- https://github.com/blacklanternsecurity/bbot/pull/788 +- https://github.com/blacklanternsecurity/bbot/pull/791 + +### New Modules +- https://github.com/blacklanternsecurity/bbot/pull/774 + +## v1.1.1 +October 11, 2023 + +Includes webhook output modules - Discord, Slack, and Teams! + +![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/72e3e940-a41a-4c7a-952e-49a1d7cae526) + +### Improvements +- https://github.com/blacklanternsecurity/bbot/pull/677 +- https://github.com/blacklanternsecurity/bbot/pull/674 +- https://github.com/blacklanternsecurity/bbot/pull/683 +- https://github.com/blacklanternsecurity/bbot/pull/740 +- https://github.com/blacklanternsecurity/bbot/pull/743 +- https://github.com/blacklanternsecurity/bbot/pull/748 +- https://github.com/blacklanternsecurity/bbot/pull/749 +- https://github.com/blacklanternsecurity/bbot/pull/751 +- https://github.com/blacklanternsecurity/bbot/pull/692 + +### Bugfixes +- https://github.com/blacklanternsecurity/bbot/pull/691 +- https://github.com/blacklanternsecurity/bbot/pull/684 +- https://github.com/blacklanternsecurity/bbot/pull/669 +- https://github.com/blacklanternsecurity/bbot/pull/664 +- https://github.com/blacklanternsecurity/bbot/pull/737 +- https://github.com/blacklanternsecurity/bbot/pull/741 +- https://github.com/blacklanternsecurity/bbot/pull/744 +- https://github.com/blacklanternsecurity/bbot/issues/760 +- https://github.com/blacklanternsecurity/bbot/issues/759 +- https://github.com/blacklanternsecurity/bbot/issues/758 +- https://github.com/blacklanternsecurity/bbot/pull/764 +- https://github.com/blacklanternsecurity/bbot/pull/773 + +### New Modules +- https://github.com/blacklanternsecurity/bbot/pull/689 +- https://github.com/blacklanternsecurity/bbot/pull/665 +- https://github.com/blacklanternsecurity/bbot/pull/663 + +## v1.1.0 +August 4, 2023 + +**New Features**: + +- Complete Asyncification +- Documentation + auto-updating pipelines +- Ability to list flags and their descriptions with `-lf` +- Fine-grained rate-limiting for HTTP and DNS + +**Improvements / Fixes**: + +- Better tests (one for each individual module, 91% test coverage) +- New and improved paramminer modules +- Misc bugfixes + +**New Modules**: + +- Git (detects exposed .git folder on websites) +- [Subdomain Center](https://www.subdomain.center/) (subdomain enumeration) +- [Columbus API](https://columbus.elmasy.com/) (subdomain enumeration) +- MySSL (subdomain enumeration) +- Sitedossier (subdomain enumeration) +- Digitorus (subdomain enumeration) +- Nmap (port scanner, more reliable than naabu) + - Naabu has been removed due to reliability issues +- NSEC (DNSSEC zone-walking for subdomain enumeration) +- OAUTH (Enumerates OAUTH / OpenID-Connect, detects sprayable endpoints) +- Azure Realm (Detects Managed/Federated Azure Tenants) +- Subdomains output module + + +## v1.0.5 +March 10, 2023 + +**New Modules**: + +- [Badsecrets](https://github.com/blacklanternsecurity/badsecrets) ([blacklist3r](https://github.com/NotSoSecure/Blacklist3r) but better!) +- Subdomain Hijacking (uses [can-i-take-over-xyz](https://github.com/EdOverflow/can-i-take-over-xyz)) +- [WafW00f](https://github.com/EnableSecurity/wafw00f) +- [Fingerprintx](https://github.com/praetorian-inc/fingerprintx) +- [Masscan](https://github.com/robertdavidgraham/masscan) +- Robots.txt +- Web Report +- IIS shortnames (Pure Python rewrite) + +**New Features**: + +- Automatic tagging of cloud resources (with [cloudcheck](https://github.com/blacklanternsecurity/cloudcheck)) +- Significant performance increases +- Bug fixes +- Better tests + code coverage +- Support for punycode (non-ascii) domains +- Better support for non-64-bit systems +- Enter key now toggles verbosity during scan + + +## v1.0.4 +December 15, 2022 + +**New Modules**: + +- Storage buckets: + - Azure + - GCP + - AWS + - DigitalOcean +- ipstack (geolocation) +- BeVigil +- ASN (rewrite) + +**New Features**: + +- Colored vulnerabilities on CLI +- Log full nuclei output +- Various bugfixes +- Better handling of: + - DNS wildcards + - Infinite DNS-record chains + - Infinite HTTP redirects +- Improved module tests + +## v1.0.3 +October 12, 2022 + +**Changes**: + +- Tag URL events with their corresponding IP address +- Automatic docker hub publishing +- Added `retries` option for httpx module +- Added `asset_inventory` output module +- Improvements to nuclei module +- Avoid unnecessary failed sudo attempts during dependency install +- Improved Python API +- Added AnubisDB module +- Various bugfixes +- Add examples to `--help` output +- Reduce annoying warnings on free API modules +- Update iis_shortnames .jar dependency +- Updated documentation to explain targets, whitelists, blacklists +- Added help for module-specific options +- Added warning if unable to validate public DNS servers (for massdns) +- Various performance optimizations +- Various bugfixes +- Fix Pypi auto-publishing +- Added bug report template +- Added examples in README +- Improved wildcard detection +- Added DNS retry functionality +- Improved excavate hostname extraction +- Added command-line option for installing all dependencies +- Improved gowitness dependency install, improved tests From 2e0582247efb21aa2f0ccd03264ef592e85eb0cb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 31 Oct 2023 13:45:58 -0400 Subject: [PATCH 46/49] fix wappalyzer tests to work on python 3.12 --- bbot/modules/wappalyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index 7bc10b448..6d30fc057 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -17,7 +17,7 @@ class wappalyzer(BaseModule): meta = { "description": "Extract technologies from web responses", } - deps_pip = ["python-Wappalyzer~=0.3.1"] + deps_pip = ["python-Wappalyzer~=0.3.1", "aiohttp~=3.9.0b0"] # accept all events regardless of scope distance scope_distance_modifier = None _max_event_handlers = 5 From bfe4d01a6d938e24d9897ebcc8665c320b106046 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 31 Oct 2023 11:26:07 -0400 Subject: [PATCH 47/49] Fix inconsistency with 404 URLs --- bbot/modules/deadly/nuclei.py | 4 +++- bbot/modules/httpx.py | 3 ++- .../module_tests/test_module_httpx.py | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 94d664b6f..df5f86140 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -175,7 +175,9 @@ def correlate_event(self, events, host): for event in events: if host in event: return event - self.warning("Failed to correlate nuclei result with event") + self.verbose(f"Failed to correlate nuclei result for {host}. Possible source events:") + for event in events: + self.verbose(f" - {event.data}") async def execute_nuclei(self, nuclei_input): command = [ diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index ff53e9972..6341f16ea 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -129,7 +129,8 @@ async def handle_batch(self, *events): continue # discard 404s from unverified URLs - if source_event.type == "URL_UNVERIFIED" and status_code in (404,): + path = j.get("path", "/") + if source_event.type == "URL_UNVERIFIED" and status_code in (404,) and path != "/": self.debug(f'Discarding 404 from "{url}"') continue diff --git a/bbot/test/test_step_2/module_tests/test_module_httpx.py b/bbot/test/test_step_2/module_tests/test_module_httpx.py index a58fb3133..fcf134dd3 100644 --- a/bbot/test/test_step_2/module_tests/test_module_httpx.py +++ b/bbot/test/test_step_2/module_tests/test_module_httpx.py @@ -48,3 +48,21 @@ def check(self, module_test, events): url = True assert url, "Failed to visit target URL" assert open_port, "Failed to visit target OPEN_TCP_PORT" + + +class TestHTTPX_404(ModuleTestBase): + targets = ["https://127.0.0.1:9999"] + modules_overrides = ["httpx", "speculate", "excavate"] + config_overrides = {"internal_modules": {"speculate": {"ports": "8888,9999"}}} + + async def setup_after_prep(self, module_test): + module_test.httpserver.expect_request("/").respond_with_data( + "Redirecting...", status=301, headers={"Location": "https://127.0.0.1:9999"} + ) + module_test.httpserver_ssl.expect_request("/").respond_with_data("404 not found", status=404) + + def check(self, module_test, events): + assert 1 == len( + [e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and "status-301" in e.tags] + ) + assert 1 == len([e for e in events if e.type == "URL" and e.data == "https://127.0.0.1:9999/"]) From c77c809cc8956dada5e0cd39573ea4d18b949d77 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 31 Oct 2023 15:42:23 -0400 Subject: [PATCH 48/49] Offload wayback URL parsing into separate process --- bbot/core/helpers/validators.py | 1 + bbot/modules/wayback.py | 25 ++++++++++++++++++++++++- bbot/scanner/scanner.py | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py index 359e431a1..5b0dfc284 100644 --- a/bbot/core/helpers/validators.py +++ b/bbot/core/helpers/validators.py @@ -221,6 +221,7 @@ def collapse_urls(urls, threshold=10): ["http://evilcorp.com/user/11111/info"] """ + log.verbose(f"Collapsing {len(urls):,} URLs") url_hashes = {} for url in urls: try: diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index d5b8f320f..92219d97c 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -1,3 +1,5 @@ +from datetime import datetime + from bbot.modules.templates.subdomain_enum import subdomain_enum @@ -49,8 +51,19 @@ async def query(self, query): except KeyError: continue + self.verbose(f"Found {len(urls):,} URLs for {query}") + dns_names = set() - for parsed_url in self.helpers.validators.collapse_urls(urls, threshold=self.garbage_threshold): + collapsed_urls = 0 + start_time = datetime.now() + parsed_urls = await self.scan.run_in_executor_mp( + self.execute_callback, + self.helpers.validators.collapse_urls, + urls, + threshold=self.garbage_threshold, + ) + for parsed_url in parsed_urls: + collapsed_urls += 1 if not self.urls: dns_name = parsed_url.hostname h = hash(dns_name) @@ -59,4 +72,14 @@ async def query(self, query): results.add((dns_name, "DNS_NAME")) else: results.add((parsed_url.geturl(), "URL_UNVERIFIED")) + end_time = datetime.now() + duration = self.helpers.human_timedelta(end_time - start_time) + self.verbose(f"Collapsed {len(urls):,} -> {collapsed_urls:,} URLs in {duration}") return results + + @staticmethod + def execute_callback(callback, *args, **kwargs): + """ + This exists so that we can run our URL parsing logic in a separate process. + """ + return list(callback(*args, **kwargs)) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index ecc4d31e4..c2ffa7288 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -255,7 +255,7 @@ def __init__( mp.set_start_method("spawn") except Exception: self.warning(f"Failed to set multiprocessing spawn method. This may negatively affect performance.") - self.process_pool = ProcessPoolExecutor() + self.process_pool = ProcessPoolExecutor(max_tasks_per_child=10) self._stopping = False From 1e8c6bd860cf8574a017d746fa8d9003e2902deb Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 31 Oct 2023 16:06:43 -0400 Subject: [PATCH 49/49] increase max tasks per child --- bbot/scanner/scanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c2ffa7288..12ac280d5 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -255,7 +255,7 @@ def __init__( mp.set_start_method("spawn") except Exception: self.warning(f"Failed to set multiprocessing spawn method. This may negatively affect performance.") - self.process_pool = ProcessPoolExecutor(max_tasks_per_child=10) + self.process_pool = ProcessPoolExecutor(max_tasks_per_child=100) self._stopping = False