From ffbda16786907fc1b65cb51a1cee7b7779a088e4 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 19:59:37 -0400 Subject: [PATCH 001/238] lightfuzz (new) initial --- bbot/core/helpers/misc.py | 19 +- bbot/modules/lightfuzz.py | 195 ++++++++++++++ bbot/modules/lightfuzz_submodules/__init__.py | 0 bbot/modules/lightfuzz_submodules/base.py | 92 +++++++ bbot/modules/lightfuzz_submodules/cmdi.py | 80 ++++++ bbot/modules/lightfuzz_submodules/crypto.py | 241 ++++++++++++++++++ bbot/modules/lightfuzz_submodules/path.py | 80 ++++++ bbot/modules/lightfuzz_submodules/sqli.py | 97 +++++++ bbot/modules/lightfuzz_submodules/ssti.py | 16 ++ bbot/modules/lightfuzz_submodules/xss.py | 78 ++++++ bbot/presets/web/lightfuzz-intense.md | 20 ++ bbot/presets/web/lightfuzz-xss.md | 14 + bbot/presets/web/lightfuzz.md | 14 + 13 files changed, 940 insertions(+), 6 deletions(-) create mode 100644 bbot/modules/lightfuzz.py create mode 100644 bbot/modules/lightfuzz_submodules/__init__.py create mode 100644 bbot/modules/lightfuzz_submodules/base.py create mode 100644 bbot/modules/lightfuzz_submodules/cmdi.py create mode 100644 bbot/modules/lightfuzz_submodules/crypto.py create mode 100644 bbot/modules/lightfuzz_submodules/path.py create mode 100644 bbot/modules/lightfuzz_submodules/sqli.py create mode 100644 bbot/modules/lightfuzz_submodules/ssti.py create mode 100644 bbot/modules/lightfuzz_submodules/xss.py create mode 100644 bbot/presets/web/lightfuzz-intense.md create mode 100644 bbot/presets/web/lightfuzz-xss.md create mode 100644 bbot/presets/web/lightfuzz.md diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index cd6bbfef1..7930967c5 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -790,14 +790,14 @@ def recursive_decode(data, max_depth=5): rand_pool = string.ascii_lowercase rand_pool_digits = rand_pool + string.digits - -def rand_string(length=10, digits=True): +def rand_string(length=10, digits=True, numeric_only=False): """ Generates a random string of specified length. Args: length (int, optional): The length of the random string. Defaults to 10. digits (bool, optional): Whether to include digits in the string. Defaults to True. + numeric_only (bool, optional): Whether to generate a numeric-only string. Defaults to False. Returns: str: A random string of the specified length. @@ -809,11 +809,18 @@ def rand_string(length=10, digits=True): 'ap4rsdtg5iw7ey7y3oa5' >>> rand_string(30, digits=False) 'xdmyxtglqfzqktngkesyulwbfrihva' + >>> rand_string(15, numeric_only=True) + '934857349857395' """ - pool = rand_pool - if digits: - pool = rand_pool_digits - return "".join([random.choice(pool) for _ in range(int(length))]) + if numeric_only: + pool = string.digits + elif digits: + pool = string.ascii_letters + string.digits + else: + pool = string.ascii_letters + + return "".join(random.choice(pool) for _ in range(length)) + def truncate_string(s, n): diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py new file mode 100644 index 000000000..7322252f1 --- /dev/null +++ b/bbot/modules/lightfuzz.py @@ -0,0 +1,195 @@ +from bbot.modules.base import BaseModule +import statistics +import re +import os +import base64 +import urllib.parse + +from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote +from bbot.errors import InteractshError, HttpCompareError + +from .lightfuzz_submodules.cmdi import CmdILightFuzz +from .lightfuzz_submodules.crypto import CryptoLightfuzz +from .lightfuzz_submodules.path import PathTraversalLightfuzz +from .lightfuzz_submodules.sqli import SQLiLightfuzz +from .lightfuzz_submodules.ssti import SSTILightfuzz +from .lightfuzz_submodules.xss import XSSLightfuzz + +class lightfuzz(BaseModule): + watched_events = ["URL", "WEB_PARAMETER"] + produced_events = ["FINDING", "VULNERABILITY"] + flags = ["active", "web-thorough"] + + submodules = { + "sqli": {"description": "SQL Injection","module": SQLiLightfuzz }, + "cmdi": {"description": "Command Injection","module": CmdILightFuzz }, + "xss": {"description": "Cross-site Scripting","module": XSSLightfuzz }, + "path": {"description": "Path Traversal","module": PathTraversalLightfuzz }, + "ssti": {"description": "Server-side Template Injection","module": SSTILightfuzz }, + "crypto": {"description": "Cryptography Probe","module": CryptoLightfuzz } + } + + options = {"force_common_headers": False, "enabled_submodules": []} + options_desc = {"force_common_headers": "Force emit commonly exploitable parameters that may be difficult to detect", "enabled_submodules": "A list of submodules to enable. Empty list enabled all modules."} + + meta = {"description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner"} + common_headers = ["x-forwarded-for", "user-agent"] + parameter_blacklist = [ + "__VIEWSTATE", + "__EVENTARGUMENT", + "__EVENTVALIDATION", + "__EVENTTARGET", + "__EVENTARGUMENT", + "__VIEWSTATEGENERATOR", + "__SCROLLPOSITIONY", + "__SCROLLPOSITIONX", + "ASP.NET_SessionId", + "JSESSIONID", + "PHPSESSID", + "__cf_bm", + ] + in_scope_only = True + + max_event_handlers = 4 + + async def setup(self): + self.event_dict = {} + self.interactsh_subdomain_tags = {} + self.interactsh_instance = None + self.enabled_submodules = self.config.get("enabled_submodules") + + for m in self.enabled_submodules: + if m not in self.submodules: + self.hugewarning(f"Invalid Lightfuzz submodule ({m}) specified in enabeld_modules") + return False + + for submodule, submodule_dict in self.submodules.items(): + if submodule in self.enabled_submodules or self.enabled_submodules == []: + setattr(self, submodule, True) + self.hugeinfo(f"Lightfuzz {submodule_dict['description']} Submodule Enabled") + + if submodule == "submodule_cmdi" and self.scan.config.get("interactsh_disable", False) == False: + try: + self.interactsh_instance = self.helpers.interactsh() + self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback) + except InteractshError as e: + self.warning(f"Interactsh failure: {e}") + else: + setattr(self, submodule, False) + return True + + async def interactsh_callback(self, r): + full_id = r.get("full-id", None) + if full_id: + if "." in full_id: + details = self.interactsh_subdomain_tags.get(full_id.split(".")[0]) + if not details["event"]: + return + await self.emit_event( + { + "severity": "CRITICAL", + "host": str(details["event"].host), + "url": details["event"].data["url"], + "description": f"OS Command Injection (OOB Interaction) Type: [{details['type']}] Parameter Name: [{details['name']}] Probe: [{details['probe']}]", + }, + "VULNERABILITY", + details["event"], + ) + else: + # this is likely caused by something trying to resolve the base domain first and can be ignored + self.debug("skipping result because subdomain tag was missing") + + def _outgoing_dedup_hash(self, event): + return hash( + ( + "lightfuzz", + str(event.host), + event.data["url"], + event.data["description"], + event.data.get("type", ""), + event.data.get("name", ""), + ) + ) + + def in_bl(self, value): + in_bl = False + for bl_param in self.parameter_blacklist: + if bl_param.lower() == value.lower(): + in_bl = True + return in_bl + + def url_unparse(self, param_type, parsed_url): + if param_type == "GETPARAM": + querystring = "" + else: + querystring = parsed_url.query + return urlunparse( + ( + parsed_url.scheme, + parsed_url.netloc, + parsed_url.path, + "", + querystring if self.retain_querystring else "", + "", + ) + ) + + async def run_submodule(self, submodule, event): + submodule_instance = submodule(self, event) + await submodule_instance.fuzz() + if len(submodule_instance.results) > 0: + for r in submodule_instance.results: + event_data = {"host": str(event.host), "url": event.data["url"], "description": r["description"]} + if r["type"] == "VULNERABILITY": + event_data["severity"] = r["severity"] + await self.emit_event( + event_data, + r["type"], + event, + ) + + async def handle_event(self, event): + + if event.type == "URL": + if self.config.get("force_common_headers", False) == False: + + return False + + for h in self.common_headers: + description = f"Speculative (Forced) Header [{h}]" + data = { + "host": str(event.host), + "type": "HEADER", + "name": h, + "original_value": None, + "url": event.data, + "description": description, + } + await self.emit_event(data, "WEB_PARAMETER", event) + + + elif event.type == "WEB_PARAMETER": + for submodule, submodule_dict in self.submodules.items(): + if getattr(self, submodule): + self.debug(f"Starting {submodule_dict['description']} fuzz()") + await self.run_submodule(submodule_dict['module'], event) + + + async def cleanup(self): + if self.interactsh_instance: + try: + await self.interactsh_instance.deregister() + self.debug( + f"successfully deregistered interactsh session with correlation_id {self.interactsh_instance.correlation_id}" + ) + except InteractshError as e: + self.warning(f"Interactsh failure: {e}") + + async def finish(self): + if self.interactsh_instance: + await self.helpers.sleep(5) + try: + for r in await self.interactsh_instance.poll(): + await self.interactsh_callback(r) + except InteractshError as e: + self.debug(f"Error in interact.sh: {e}") \ No newline at end of file diff --git a/bbot/modules/lightfuzz_submodules/__init__.py b/bbot/modules/lightfuzz_submodules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py new file mode 100644 index 000000000..dbdb93d49 --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -0,0 +1,92 @@ +class BaseLightfuzz: + def __init__(self, lightfuzz, event): + self.lightfuzz = lightfuzz + self.event = event + self.results = [] + + async def send_probe(self, probe): + getparams = {self.event.data["name"]: probe} + url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams).geturl() + self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}") + r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10) + if r: + return r.text + + def compare_baseline(self, event_type, probe, cookies): + + if event_type == "GETPARAM": + baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" + http_compare = self.lightfuzz.helpers.http_compare( + baseline_url, cookies=cookies, include_cache_buster=True + ) + elif event_type == "COOKIE": + cookies_probe = {self.event.data["name"]: f"{probe}"} + http_compare = self.lightfuzz.helpers.http_compare( + self.event.data["url"], include_cache_buster=False, cookies={**cookies, **cookies_probe} + ) + elif event_type == "HEADER": + headers = {self.event.data["name"]: f"{probe}"} + http_compare = self.lightfuzz.helpers.http_compare( + self.event.data["url"], include_cache_buster=False, headers=headers, cookies=cookies + ) + elif event_type == "POSTPARAM": + data = {self.event.data["name"]: f"{probe}"} + if self.event.data["additional_params"] is not None: + data.update(self.event.data["additional_params"]) + http_compare = self.lightfuzz.helpers.http_compare( + self.event.data["url"], method="POST", include_cache_buster=False, data=data, cookies=cookies + ) + return http_compare + + async def compare_probe(self, http_compare, event_type, probe, cookies): + + if event_type == "GETPARAM": + probe_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" + compare_result = await http_compare.compare(probe_url, cookies=cookies) + elif event_type == "COOKIE": + cookies_probe = {self.event.data["name"]: probe} + compare_result = await http_compare.compare(self.event.data["url"], cookies={**cookies, **cookies_probe}) + elif event_type == "HEADER": + headers = {self.event.data["name"]: f"{probe}"} + compare_result = await http_compare.compare(self.event.data["url"], headers=headers, cookies=cookies) + elif event_type == "POSTPARAM": + data = {self.event.data["name"]: f"{probe}"} + if self.event.data["additional_params"] is not None: + data.update(self.event.data["additional_params"]) + compare_result = await http_compare.compare( + self.event.data["url"], method="POST", data=data, cookies=cookies + ) + return compare_result + + async def standard_probe(self, event_type, cookies, probe_value, timeout=10): + + method = "GET" + if event_type == "GETPARAM": + url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}" + else: + url = self.event.data["url"] + if event_type == "COOKIE": + cookies_probe = {self.event.data["name"]: probe_value} + cookies = {**cookies, **cookies_probe} + if event_type == "HEADER": + headers = {self.event.data["name"]: probe_value} + else: + headers = {} + if event_type == "POSTPARAM": + method = "POST" + data = {self.event.data["name"]: probe_value} + if self.event.data["additional_params"] is not None: + data.update(self.event.data["additional_params"]) + else: + data = {} + self.lightfuzz.debug(f"standard_probe requested URL: [{url}]") + return await self.lightfuzz.helpers.request( + method=method, + cookies=cookies, + headers=headers, + data=data, + url=url, + allow_redirects=False, + retries=0, + timeout=timeout, + ) \ No newline at end of file diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py new file mode 100644 index 000000000..c6275f43d --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -0,0 +1,80 @@ +from bbot.errors import HttpCompareError +from .base import BaseLightfuzz + +import urllib.parse + +class CmdILightFuzz(BaseLightfuzz): + + async def fuzz(self): + + cookies = self.event.data.get("assigned_cookies", {}) + if ( + "original_value" in self.event.data + and self.event.data["original_value"] is not None + and len(self.event.data["original_value"]) != 0 + ): + probe_value = self.event.data["original_value"] + else: + probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + + canary = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) + + cmdi_probe_strings = [ + "AAAA", + ";", + "&&", + "||", + "&", + "|", + ] + + positive_detections = [] + for p in cmdi_probe_strings: + try: + echo_probe = f"{probe_value}{p} echo {canary} {p}" + if self.event.data["type"] == "GETPARAM": + echo_probe = urllib.parse.quote(echo_probe.encode(), safe="") + cmdi_probe = await self.compare_probe(http_compare, self.event.data["type"], echo_probe, cookies) + if cmdi_probe[3]: + if canary in cmdi_probe[3].text and "echo" not in cmdi_probe[3].text: + self.lightfuzz.debug(f"canary [{canary}] found in response when sending probe [{p}]") + if p == "AAAA": + self.lightfuzz.hugewarning( + f"False Postive Probe appears to have been triggered for {self.event.data['url']}, aborting remaining detection" + ) + return + positive_detections.append(p) + except HttpCompareError as e: + self.lightfuzz.debug(e) + continue + + if len(positive_detections) > 0: + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE OS Command Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [echo canary] CMD Probe Delimeters: [{' '.join(positive_detections)}]", + } + ) + + # Blind OS Command Injection + if self.lightfuzz.interactsh_instance: + self.lightfuzz.event_dict[self.event.data["url"]] = self.event + + for p in cmdi_probe_strings: + + subdomain_tag = self.lightfuzz.helpers.rand_string(4, digits=False) + self.lightfuzz.interactsh_subdomain_tags[subdomain_tag] = { + "event": self.event, + "type": self.event.data["type"], + "name": self.event.data["name"], + "probe": p, + } + interactsh_probe = f"{p} nslookup {subdomain_tag}.{self.lightfuzz.interactsh_domain} {p}" + + if self.event.data["type"] == "GETPARAM": + interactsh_probe = urllib.parse.quote(interactsh_probe.encode(), safe="") + await self.standard_probe( + self.event.data["type"], cookies, f"{probe_value}{interactsh_probe}", timeout=15 + ) + diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py new file mode 100644 index 000000000..415ccde44 --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -0,0 +1,241 @@ +from .base import BaseLightfuzz + +from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote + +class CryptoLightfuzz(BaseLightfuzz): + + @staticmethod + def is_hex(s): + try: + bytes.fromhex(s) + return True + except ValueError: + return False + + @staticmethod + def is_base64(s): + try: + if base64.b64encode(base64.b64decode(s)).decode() == s: + return True + except Exception: + return False + return False + + crypto_error_strings = [ + "invalid mac", + "padding is invalid and cannot be removed", + "bad data", + "length of the data to decrypt is invalid", + "specify a valid key size", + "invalid algorithm specified", + "object already exists", + "key does not exist", + "the parameter is incorrect", + "cryptography exception", + "access denied", + "unknown error", + "invalid provider type", + "no valid cert found", + "cannot find the original signer", + "signature description could not be created", + "crypto operation failed", + "OpenSSL Error", + ] + + @staticmethod + def format_agnostic_decode(input_string): + encoding = "unknown" + decoded_input = unquote(input_string) + if CryptoLightfuzz.is_hex(decoded_input): + data = bytes.fromhex(decoded_input) + encoding = "hex" + elif CryptoLightfuzz.is_base64(decoded_input): + data = base64.b64decode(decoded_input) + encoding = "base64" + else: + data = str + return data, encoding + + + @staticmethod + def format_agnostic_encode(data, encoding): + if encoding == "hex": + encoded_data = data.hex() + elif encoding == "base64": + encoded_data = base64.b64encode(data).decode('utf-8') # base64 encoding returns bytes, decode to string + else: + raise ValueError("Unsupported encoding type specified") + return encoded_data + + @staticmethod + def modify_string(input_string, action="truncate", position=None, extension_length=1): + + data, encoding = CryptoLightfuzz.format_agnostic_decode(input_string) + if encoding != "base64" and encoding != "hex": + raise ValueError("Input must be either hex or base64 encoded") + + if action == "truncate": + modified_data = data[:-1] # Remove the last byte + elif action == "mutate": + if not position: + position = len(data) // 2 + if position < 0 or position >= len(data): + raise ValueError("Position out of range") + byte_list = list(data) + byte_list[position] = (byte_list[position] + 1) % 256 + modified_data = bytes(byte_list) + elif action == "extend": + modified_data = data + (b"\x00" * extension_length) + elif action == "flip": + if not position: + position = len(data) // 2 + if position < 0 or position >= len(data): + raise ValueError("Position out of range") + byte_list = list(data) + byte_list[position] ^= 0xFF # Flip all bits in the byte at the specified position + modified_data = bytes(byte_list) + else: + raise ValueError("Unsupported action") + return CryptoLightfuzz.format_agnostic_encode(modified_data, encoding) + + def is_likely_encrypted(self, data, threshold=5.5): + entropy = self.lightfuzz.helpers.calculate_entropy(data) + return entropy >= threshold + + def cryptanalysis(self, input_string): + + likely_crypto = False + possible_block_cipher = False + data, encoding = self.format_agnostic_decode(input_string) + likely_crypto = self.is_likely_encrypted(data) + data_length = len(data) + if data_length % 8 == 0: + possible_block_cipher = True + return likely_crypto, possible_block_cipher + + async def padding_oracle_execute(self, data, encoding, cookies, possible_first_byte=False): + if possible_first_byte: + baseline_byte = b'\xFF' + starting_pos = 0 + else: + baseline_byte = b'\x00' + starting_pos = 1 + + baseline = self.compare_baseline(self.event.data["type"], data[:-1] + baseline_byte, cookies) + differ_count = 0 + for i in range(starting_pos, starting_pos+254): + byte = bytes([i]) + oracle_probe = await self.compare_probe(baseline, self.event.data["type"], self.format_agnostic_encode(data[:-1] + byte, encoding), cookies) + if oracle_probe[0] == False and "body" in oracle_probe[1]: + differ_count += 1 + if i == 1: + possible_first_byte = True + continue + elif i == 2 and possible_first_byte == True: + # Thats two results which appear "different". Its entirely possible \x00 was the correct padding. We will break from this loop and redo it with the last byte as the baseline instead of the first + return None + if differ_count == 1: + return True + else: + return False + + async def padding_oracle(self, probe_value, cookies): + data, encoding = self.format_agnostic_decode(probe_value) + + padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies) + if padding_oracle_result == None: + self.lightfuzz.hugewarning("ENDED UP IN POSSIBLE_FIRST_BYTE SITUATION") + padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies, possible_first_byte=False) + + if padding_oracle_result == True: + context = f"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]" + self.results.append( + { + "type": "VULNERABILITY", + "severity": "HIGH", + "description": f"Padding Oracle Vulnerability. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]", + "context": context, + } + ) + + async def error_string_search(self, text_dict): + + matching_techniques = set() + matching_strings = set() + + for label, text in text_dict.items(): + matched_strings = self.lightfuzz.helpers.string_scan(self.crypto_error_strings, text) + for m in matched_strings: + matching_strings.add(m) + matching_techniques.add(label) + context = f"Lightfuzz Cryptographic Probe Submodule detected a cryptographic error after manipulating parameter: [{self.event.data['name']}]" + if len(matching_strings) > 0: + self.results.append( + { + "type": "FINDING", + "description": f"Possible Cryptographic Error. Parameter: [{self.event.data['name']}] Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]", + "context": context, + } + ) + + async def fuzz(self): + cookies = self.event.data.get("assigned_cookies", {}) + if ( + "original_value" in self.event.data + and self.event.data["original_value"] is not None + and self.event.data["original_value"] != "1" + ): + probe_value = self.event.data["original_value"] + else: + self.lightfuzz.critical( + f"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" + ) + return + + try: + truncate_probe_value = self.modify_string(probe_value, action="truncate") + mutate_probe_value = self.modify_string(probe_value, action="mutate") + except ValueError as e: + self.lightfuzz.critical(f"Encountered error modifying value: {e}, aborting") + return + + # Basic crypanalysis + likely_crypto, possible_block_cipher = self.cryptanalysis(probe_value) + if not likely_crypto: + self.lightfuzz.debug("Parameter value does not appear to be cryptographic, aborting tests") + return + + http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) + + # Cryptographic Response Divergence Test + arbitrary_probe = await self.compare_probe(http_compare, self.event.data["type"], "AAAAAAA", cookies) + truncate_probe = await self.compare_probe(http_compare, self.event.data["type"], truncate_probe_value, cookies) + mutate_probe = await self.compare_probe(http_compare, self.event.data["type"], mutate_probe_value, cookies) + + confirmed_techniques = [] + if mutate_probe[0] == False and "body" in mutate_probe[1]: + if http_compare.compare_body(mutate_probe[3].text, arbitrary_probe[3].text) == False: + confirmed_techniques.append("Single-byte Mutation") + + if mutate_probe[0] == False and "body" in mutate_probe[1]: + if http_compare.compare_body(truncate_probe[3].text, arbitrary_probe[3].text) == False: + confirmed_techniques.append("Data Truncation") + + if confirmed_techniques: + context = f"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) to appears to drive a cryptographic operation" + self.results.append( + { + "type": "FINDING", + "description": f"Probable Cryptographic Parameter [{self.event.data['name']}] Detection Technique(s): [{','.join(confirmed_techniques)}]", + "context": context, + } + ) + + # Cryptographic Error String Test + await self.error_string_search( + {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text} + ) + + # Padding Oracle Test + if possible_block_cipher: + await self.padding_oracle(probe_value, cookies) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py new file mode 100644 index 000000000..c9357fd9f --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -0,0 +1,80 @@ +from .base import BaseLightfuzz + +import urllib.parse + +class PathTraversalLightfuzz(BaseLightfuzz): + + async def fuzz(self): + cookies = self.event.data.get("assigned_cookies", {}) + if ( + "original_value" in self.event.data + and self.event.data["original_value"] is not None + and self.event.data["original_value"] != "1" + ): + probe_value = self.event.data["original_value"] + else: + self.lightfuzz.debug( + f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" + ) + return + + http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) + + # Single dot traversal tolerance test + + path_techniques = { + "single-dot traversal tolerance (no-encoding)": { + "singledot_payload": f"/./{probe_value}", + "doubledot_payload": f"/../{probe_value}", + }, + "single-dot traversal tolerance (url-encoding)": { + "singledot_payload": urllib.parse.quote(f"/./{probe_value}".encode(), safe=""), + "doubledot_payload": urllib.parse.quote(f"/../{probe_value}".encode(), safe=""), + }, + } + + for path_technique, payloads in path_techniques.items(): + + try: + singledot_probe = await self.compare_probe( + http_compare, self.event.data["type"], payloads["singledot_payload"], cookies + ) + doubledot_probe = await self.compare_probe( + http_compare, self.event.data["type"], payloads["doubledot_payload"], cookies + ) + + if ( + singledot_probe[0] == True + and doubledot_probe[0] == False + and doubledot_probe[3] != None + and not str(doubledot_probe[3].status_code).startswith("4") + and doubledot_probe[1] != ["header"] + ): + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [{path_technique}]", + } + ) + # no need to report both techniques if they both work + break + except HttpCompareError as e: + self.lightfuzz.debug(e) + continue + + # Absolute path test + + absolute_paths = {r"c:\\windows\\win.ini": "; for 16-bit app support", "/etc/passwd": "daemon:x:"} + + for path, trigger in absolute_paths.items(): + r = await self.standard_probe(self.event.data["type"], cookies, path) + if r and trigger in r.text: + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Absolute Path: {path}]", + } + ) + + + diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py new file mode 100644 index 000000000..9e4c76d67 --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -0,0 +1,97 @@ +import statistics + +from .base import BaseLightfuzz + +class SQLiLightfuzz(BaseLightfuzz): + expected_delay = 5 + + def evaluate_delay(self, mean_baseline, measured_delay): + margin = 1 + if ( + mean_baseline + self.expected_delay - margin + <= measured_delay + <= mean_baseline + self.expected_delay + margin + ): + return True + # check for exactly twice the delay, in case the statement gets placed in the query twice + elif ( + mean_baseline + (self.expected_delay * 2) - margin + <= measured_delay + <= mean_baseline + (self.expected_delay * 2) + margin + ): + return True + else: + return False + + async def fuzz(self): + + cookies = self.event.data.get("assigned_cookies", {}) + if "original_value" in self.event.data and self.event.data["original_value"] is not None: + probe_value = self.event.data["original_value"] + else: + probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) + + try: + single_quote = await self.compare_probe(http_compare, self.event.data["type"], f"{probe_value}'", cookies) + double_single_quote = await self.compare_probe( + http_compare, self.event.data["type"], f"{probe_value}''", cookies + ) + + if "code" in single_quote[1] and "code" not in double_single_quote[1]: + self.results.append( + { + "type": "FINDING", + "description": f"Possible SQL Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Single Quote/Two Single Quote]", + } + ) + except HttpCompareError as e: + self.lightfuzz.debug(e) + + standard_probe_strings = [ + f"'||pg_sleep({str(self.expected_delay)})--", # postgres + f"1' AND (SLEEP({str(self.expected_delay)})) AND '", # mysql + f"' AND (SELECT FROM DBMS_LOCK.SLEEP({str(self.expected_delay)})) AND '1'='1" # oracle (not tested) + f"; WAITFOR DELAY '00:00:{str(self.expected_delay)}'--", # mssql (not tested) + ] + method = "GET" + + baseline_1 = await self.standard_probe(self.event.data["type"], cookies, probe_value) + baseline_2 = await self.standard_probe(self.event.data["type"], cookies, probe_value) + + if baseline_1 and baseline_2: + baseline_1_delay = baseline_1.elapsed.total_seconds() + baseline_2_delay = baseline_2.elapsed.total_seconds() + mean_baseline = statistics.mean([baseline_1_delay, baseline_2_delay]) + + for p in standard_probe_strings: + confirmations = 0 + for i in range(0, 3): + r = await self.standard_probe(self.event.data["type"], cookies, f"{probe_value}{p}") + if not r: + self.lightfuzz.debug("delay measure request failed") + break + + d = r.elapsed.total_seconds() + self.lightfuzz.debug(f"measured delay: {str(d)}") + if self.evaluate_delay(mean_baseline, d): + confirmations += 1 + self.lightfuzz.debug( + f"{self.event.data['url']}:{self.event.data['name']}:{self.event.data['type']} Increasing confirmations, now: {str(confirmations)} " + ) + else: + break + + if confirmations == 3: + self.results.append( + { + "type": "FINDING", + "description": f"Possible Blind SQL Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Delay Probe ({p})]", + } + ) + + else: + self.lightfuzz.debug("Could not get baseline for time-delay tests") + + + diff --git a/bbot/modules/lightfuzz_submodules/ssti.py b/bbot/modules/lightfuzz_submodules/ssti.py new file mode 100644 index 000000000..d89cb72d7 --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/ssti.py @@ -0,0 +1,16 @@ +from .base import BaseLightfuzz + +class SSTILightfuzz(BaseLightfuzz): + async def fuzz(self): + cookies = self.event.data.get("assigned_cookies", {}) + probe_value = "<%25%3d%201337*1337%20%25>" + r = await self.standard_probe(self.event.data["type"], cookies, probe_value) + if r and "1787569" in r.text: + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Server-side Template Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Integer Multiplication]", + } + ) + + diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py new file mode 100644 index 000000000..55a1d0110 --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -0,0 +1,78 @@ +from .base import BaseLightfuzz + +import re + +class XSSLightfuzz(BaseLightfuzz): + def determine_context(self, html, random_string): + between_tags = False + in_tag_attribute = False + in_javascript = False + + between_tags_regex = re.compile(rf"<(\/?\w+)[^>]*>.*?{random_string}.*?<\/?\w+>") + in_tag_attribute_regex = re.compile(rf'<(\w+)\s+[^>]*?(\w+)="([^"]*?{random_string}[^"]*?)"[^>]*>') + in_javascript_regex = re.compile( + rf"]*>(?:(?!<\/script>)[\s\S])*?{random_string}(?:(?!<\/script>)[\s\S])*?<\/script>" + ) + + between_tags_match = re.search(between_tags_regex, html) + if between_tags_match: + between_tags = True + + in_tag_attribute_match = re.search(in_tag_attribute_regex, html) + if in_tag_attribute_match: + in_tag_attribute = True + + in_javascript_regex = re.search(in_javascript_regex, html) + if in_javascript_regex: + in_javascript = True + + return between_tags, in_tag_attribute, in_javascript + + async def check_probe(self, probe, match, context): + probe_result = await self.send_probe(probe) + if probe_result and match in probe_result: + self.results.append( + { + "type": "FINDING", + "description": f"Possible Reflected XSS. Parameter: [{self.event.data['name']}] Context: [{context}]", + } + ) + + async def fuzz(self): + lightfuzz_event = self.event.parent + + # If this came from paramminer_getparams and didn't have a http_reflection tag, we don't need to check again + if ( + lightfuzz_event.type == "WEB_PARAMETER" + and lightfuzz_event.parent.type == "paramminer_getparams" + and "http_reflection" not in lightfuzz_event.tags + ): + return + + reflection = None + random_string = self.lightfuzz.helpers.rand_string(8) + reflection_probe_result = await self.send_probe(random_string) + if reflection_probe_result and random_string in reflection_probe_result: + reflection = True + + if not reflection or reflection == False: + return + + between_tags, in_tag_attribute, in_javascript = self.determine_context(reflection_probe_result, random_string) + + self.lightfuzz.debug( + f"determine_context returned: between_tags [{between_tags}], in_tag_attribute [{in_tag_attribute}], in_javascript [{in_javascript}]" + ) + + if between_tags: + between_tags_probe = f"{random_string}" + await self.check_probe(between_tags_probe, between_tags_probe, "Between Tags") + + if in_tag_attribute: + in_tag_attribute_probe = f'{random_string}"' + in_tag_attribute_match = f'"{random_string}""' + await self.check_probe(in_tag_attribute_probe, in_tag_attribute_match, "Tag Attribute") + + if in_javascript: + in_javascript_probe = rf"" + await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-intense.md b/bbot/presets/web/lightfuzz-intense.md new file mode 100644 index 000000000..5f90f95d2 --- /dev/null +++ b/bbot/presets/web/lightfuzz-intense.md @@ -0,0 +1,20 @@ +description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques + +flags: + - web-paramminer + +modules: + - httpx + - lightfuzz + - robots + - badsecrets + +config: + url_querystring_remove: False + url_querystring_collapse: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + force_common_headers: True + retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-xss.md b/bbot/presets/web/lightfuzz-xss.md new file mode 100644 index 000000000..5fcb3117e --- /dev/null +++ b/bbot/presets/web/lightfuzz-xss.md @@ -0,0 +1,14 @@ +description: Discovery web parameters and lightly fuzz them for xss vulnerabilities +modules: + - httpx + - lightfuzz + - paramminer_getparams + +config: + url_querystring_remove: False + url_querystring_collapse: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + submodules_enabled: [xss] \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.md b/bbot/presets/web/lightfuzz.md new file mode 100644 index 000000000..0ccb740d3 --- /dev/null +++ b/bbot/presets/web/lightfuzz.md @@ -0,0 +1,14 @@ +description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques + +modules: + - httpx + - lightfuzz + - badsecrets + +config: + url_querystring_remove: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + submodules_enabled: [cmdi,crypto,sqli,ssti,xss] From 8edca2b564daf794f7ae193ee70715115b8792c2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 20:09:05 -0400 Subject: [PATCH 002/238] making WEB_PARAMETER invisible by default --- bbot/defaults.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 2dc52482c..80448b088 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -165,6 +165,7 @@ omit_event_types: - URL_UNVERIFIED - DNS_NAME_UNRESOLVED - FILESYSTEM + - WEB_PARAMETER # - IP_ADDRESS # Custom interactsh server settings From 51da74aa077636f7d0da4822bbc0031b30f4dfb4 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 20:19:58 -0400 Subject: [PATCH 003/238] fixing presets --- bbot/presets/web/lightfuzz-intense.yml | 20 ++++++++++++++++++++ bbot/presets/web/lightfuzz-xss.yml | 14 ++++++++++++++ bbot/presets/web/lightfuzz.yml | 14 ++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 bbot/presets/web/lightfuzz-intense.yml create mode 100644 bbot/presets/web/lightfuzz-xss.yml create mode 100644 bbot/presets/web/lightfuzz.yml diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml new file mode 100644 index 000000000..5f90f95d2 --- /dev/null +++ b/bbot/presets/web/lightfuzz-intense.yml @@ -0,0 +1,20 @@ +description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques + +flags: + - web-paramminer + +modules: + - httpx + - lightfuzz + - robots + - badsecrets + +config: + url_querystring_remove: False + url_querystring_collapse: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + force_common_headers: True + retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-xss.yml b/bbot/presets/web/lightfuzz-xss.yml new file mode 100644 index 000000000..5fcb3117e --- /dev/null +++ b/bbot/presets/web/lightfuzz-xss.yml @@ -0,0 +1,14 @@ +description: Discovery web parameters and lightly fuzz them for xss vulnerabilities +modules: + - httpx + - lightfuzz + - paramminer_getparams + +config: + url_querystring_remove: False + url_querystring_collapse: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + submodules_enabled: [xss] \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml new file mode 100644 index 000000000..0ccb740d3 --- /dev/null +++ b/bbot/presets/web/lightfuzz.yml @@ -0,0 +1,14 @@ +description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques + +modules: + - httpx + - lightfuzz + - badsecrets + +config: + url_querystring_remove: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + submodules_enabled: [cmdi,crypto,sqli,ssti,xss] From cb1712fc44dd126baed7c2d28a07b6aa9512c95e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 20:21:46 -0400 Subject: [PATCH 004/238] fixing presets... --- bbot/presets/web/lightfuzz-intense.md | 20 -------------------- bbot/presets/web/lightfuzz-xss.md | 14 -------------- bbot/presets/web/lightfuzz.md | 14 -------------- 3 files changed, 48 deletions(-) delete mode 100644 bbot/presets/web/lightfuzz-intense.md delete mode 100644 bbot/presets/web/lightfuzz-xss.md delete mode 100644 bbot/presets/web/lightfuzz.md diff --git a/bbot/presets/web/lightfuzz-intense.md b/bbot/presets/web/lightfuzz-intense.md deleted file mode 100644 index 5f90f95d2..000000000 --- a/bbot/presets/web/lightfuzz-intense.md +++ /dev/null @@ -1,20 +0,0 @@ -description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques - -flags: - - web-paramminer - -modules: - - httpx - - lightfuzz - - robots - - badsecrets - -config: - url_querystring_remove: False - url_querystring_collapse: False - web_spider_distance: 4 - web_spider_depth: 5 - modules: - lightfuzz: - force_common_headers: True - retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-xss.md b/bbot/presets/web/lightfuzz-xss.md deleted file mode 100644 index 5fcb3117e..000000000 --- a/bbot/presets/web/lightfuzz-xss.md +++ /dev/null @@ -1,14 +0,0 @@ -description: Discovery web parameters and lightly fuzz them for xss vulnerabilities -modules: - - httpx - - lightfuzz - - paramminer_getparams - -config: - url_querystring_remove: False - url_querystring_collapse: False - web_spider_distance: 4 - web_spider_depth: 5 - modules: - lightfuzz: - submodules_enabled: [xss] \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.md b/bbot/presets/web/lightfuzz.md deleted file mode 100644 index 0ccb740d3..000000000 --- a/bbot/presets/web/lightfuzz.md +++ /dev/null @@ -1,14 +0,0 @@ -description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques - -modules: - - httpx - - lightfuzz - - badsecrets - -config: - url_querystring_remove: False - web_spider_distance: 4 - web_spider_depth: 5 - modules: - lightfuzz: - submodules_enabled: [cmdi,crypto,sqli,ssti,xss] From 66cdb5e9a1dfb79ce306599e91bf4106173dfac4 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 20:37:29 -0400 Subject: [PATCH 005/238] fixing error imports --- bbot/modules/lightfuzz_submodules/path.py | 1 + bbot/modules/lightfuzz_submodules/sqli.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index c9357fd9f..3f981a3ff 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -1,4 +1,5 @@ from .base import BaseLightfuzz +from bbot.errors import HttpCompareError import urllib.parse diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 9e4c76d67..d44da88b3 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -1,6 +1,7 @@ -import statistics - from .base import BaseLightfuzz +from bbot.errors import HttpCompareError + +import statistics class SQLiLightfuzz(BaseLightfuzz): expected_delay = 5 From 60b909efd333cda8133819e3f9783de084a742bc Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 21:32:14 -0400 Subject: [PATCH 006/238] adding helpers --- bbot/core/helpers/misc.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 7930967c5..225fd7883 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2743,3 +2743,32 @@ def clean_dict(d, *key_names, fuzzy=False, exclude_keys=None, _prev_key=None): continue d[key] = clean_dict(val, *key_names, fuzzy=fuzzy, _prev_key=key, exclude_keys=exclude_keys) return d + + +def string_scan(substrings, text, case_insensitive=True): + automaton = ahocorasick.Automaton() + if case_insensitive: + substrings = [s.lower() for s in substrings] + text = text.lower() + for idx, substring in enumerate(substrings): + automaton.add_word(substring, (idx, substring)) + automaton.make_automaton() + found_substrings = [] + for end_index, (insert_order, original_value) in automaton.iter(text): + found_substrings.append(original_value) + return found_substrings + + +def calculate_entropy(data): + """Calculate the Shannon entropy of a byte sequence""" + if not data: + return 0 + frequency = {} + for byte in data: + if byte in frequency: + frequency[byte] += 1 + else: + frequency[byte] = 1 + data_len = len(data) + entropy = -sum((count / data_len) * math.log2(count / data_len) for count in frequency.values()) + return entropy \ No newline at end of file From b38fa534148b2d46eeee342c487341f6fa104e1a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 22:30:54 -0400 Subject: [PATCH 007/238] missing import --- bbot/core/helpers/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 225fd7883..446166a0e 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2,6 +2,7 @@ import sys import copy import json +import math import random import string import asyncio From 2f257dcd401d71b6d7c9df734a095055ed6b488a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 27 Jun 2024 17:21:54 -0400 Subject: [PATCH 008/238] yara docs initial --- docs/modules/custom_yara_rules.md | 33 ++++++++++++ docs/modules/internal_modules.md | 85 +++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 docs/modules/custom_yara_rules.md create mode 100644 docs/modules/internal_modules.md diff --git a/docs/modules/custom_yara_rules.md b/docs/modules/custom_yara_rules.md new file mode 100644 index 000000000..087b84be6 --- /dev/null +++ b/docs/modules/custom_yara_rules.md @@ -0,0 +1,33 @@ +# Custom Yara Rules + +### Overview +Though the `excavate` interanal module, BBOT supports searching through HTTP response data using custom YARA rules. + +This feature can be utilized with the command line option `--custom-yara-rules` or `-cy`, followed by a file containing the YARA rules. + + +### Custom options + +BBOT supports the use of a few custom `meta` attributes within YARA rules, which will alter the behavior of the rule and the post-processing of the results. + +#### description + +The description of the rule. Will end up in the description of any produced events if defined. + +#### tags + +Tags specified with this option will be passed-on to any resulting emitted events. Provided as a comma separated string, as shown below: + +TBA + +#### emit_match + +When set to True, the contents returned from a successful extraction via a YARA regex will be included in the FINDING event which is emitted. + +Consider the following example: + +TBA + +### YARA Resources + +TBA \ No newline at end of file diff --git a/docs/modules/internal_modules.md b/docs/modules/internal_modules.md new file mode 100644 index 000000000..d9372fac4 --- /dev/null +++ b/docs/modules/internal_modules.md @@ -0,0 +1,85 @@ +# List of Modules + +## What are internal modules? + +Internal modules just like regular modules, except that they run all the time. They do not have to be explicitly enabled. They can, however, be explicitly disabled if needed. + +Turning them off is simple, a root-level config option is present which can be set to False to disable them: + +``` +# Infer certain events from others, e.g. IPs from IP ranges, DNS_NAMEs from URLs, etc. +speculate: True +# Passively search event data for URLs, hostnames, emails, etc. +excavate: True +# Summarize activity at the end of a scan +aggregate: True +# DNS resolution +dnsresolve: True +# Cloud provider tagging +cloudcheck: True +``` + +These modules are executing core functionality that is normally essential for a typical BBOT scan. Let's take a quick look at each one's functionality: + +### aggregate + +Summarize statistics at the end of a scan. Disable if you don't want to see this table. + +### cloud + +The cloud module looks at events and tries to determine if they are associated with a cloud provider and tags them as such, and can also identify certain cloud resources + +### dns + +The DNS internal module controls the basic DNS resoultion the BBOT performs, and all of the supporting machinery like wildcard detection, etc. + +### excavate + +The excavate internal module designed to passively extract valuable information from HTTP response data. It primarily uses YARA regexes to extract information, with various events being produced from the post-processing of the YARA results. + +Here is a summary of the data it produces: + +#### URLs + +By extracting URLs from all visited pages, this is actually already half of a web-spider. The other half is recursion, which is baked in to BBOT from the ground up. Therefore, protections are in place by default in the form of `web_spider_distance` and `web_spider_depth` settings. These settings govern restrictions to URLs recursively harvested from HTTP responses, preventing endless runaway scans. However, in the right situation the controlled use of a web-spider is extremely powerful. + +#### Parameter Extraction + +Parameter Extraction +The parameter extraction functionality identifies and extracts key web parameters from HTTP responses, and produced `WEB_PARAMETER` events. This includes parameters found in GET and POST requests, HTML forms, and jQuery requests. Currently, these are only used by the `hunt` module, and by the `paramminer` modules, to a limited degree. However, future functionality will make extensive use of these events. + +#### Email Extraction + +Detect email addresses within HTTP_RESPONSE data. + +#### Error Detection + +Scans for verbose error messages in HTTP responses and raw text data. By identifying specific error signatures from various programming languages and frameworks, this feature helps uncover misconfigurations, debugging information, and potential vulnerabilities. This insight is invaluable for identifying weak points or anomalies in web applications. + +#### Content Security Policy (CSP) Extraction +The CSP extraction capability focuses on extracting domains from Content-Security-Policy headers. By analyzing these headers, BBOT can identify additional domains which can get fed back into the scan. + +#### Serialization Detection +Serialized objects are a common source of serious security vulnerablities. Excavate aims to detect those used in Java, .NET, and PHP applications. + +#### Functionality Detection +Looks for specific web functionalities such as file upload fields and WSDL URLs. By identifying these elements, BBOT can pinpoint areas of the application that may require further scrutiny for security vulnerabilities. + +#### Non-HTTP Scheme Detection +The non-HTTP scheme detection capability extracts URLs with non-HTTP schemes, such as ftp, mailto, and javascript. By identifying these URLs, BBOT can uncover additional vectors for attack or information leakage. + +#### Custom Yara Rules + +Excavate supports the use of custom YARA rules, which wil be added to the other rules before the scan start. For more info, view this. + +### speculate + +Speculate is all about inferring one data type from another, particularly when certain tools like port scanners are not enabled. This is essential functionality for most BBOT scans, allowing for the discovery of web resources when starting with a DNS-only target list without a port scanner. It bridges gaps in the data, providing a more comprehensive view of the target by leveraging existing information. + +* IP_RANGE: Converts an IP range into individual IP addresses and emits them as IP_ADDRESS events. +* DNS_NAME: Generates parent domains from DNS names. +* URL and URL_UNVERIFIED: Infers open TCP ports from URLs and speculates on sub-directory URLs. +* General URL Speculation: Emits URL_UNVERIFIED events for URLs not already in the event's history. +* IP_ADDRESS / DNS_NAME: Infers open TCP ports if active port scanning is not enabled. +* ORG_STUB: Derives organization stubs from TLDs, social stubs, or Azure tenant names and emits them as ORG_STUB events. +* USERNAME: Converts usernames to email addresses if they validate as such. \ No newline at end of file From 95bb8c517edf0e80531d6c8d51cc56f0eb1d43d6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 27 Jun 2024 17:23:58 -0400 Subject: [PATCH 009/238] fix double printing of bbot logo --- bbot/cli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index 686e4216e..f3788f61a 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -2,15 +2,17 @@ import sys import logging +import multiprocessing from bbot.errors import * from bbot import __version__ from bbot.logger import log_to_stderr -silent = "-s" in sys.argv or "--silent" in sys.argv +if multiprocessing.current_process().name == "MainProcess": + silent = "-s" in sys.argv or "--silent" in sys.argv -if not silent: - ascii_art = f"""  ______  _____ ____ _______ + if not silent: + ascii_art = rf"""  ______  _____ ____ _______ | ___ \| __ \ / __ \__ __| | |___) | |__) | | | | | | | ___ <| __ <| | | | | | @@ -18,9 +20,9 @@ |______/|_____/ \____/ |_| BIGHUGE BLS OSINT TOOL {__version__} - www.blacklanternsecurity.com/bbot +www.blacklanternsecurity.com/bbot """ - print(ascii_art, file=sys.stderr) + print(ascii_art, file=sys.stderr) scan_name = "" From 709f4bb814f0f474f7811e47da6cb18662a3b2ba Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 15:46:43 +0000 Subject: [PATCH 010/238] tweak to custom yara messaging --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index c1fcda171..9d4c48f3c 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -75,7 +75,7 @@ def __init__(self, excavate): async def preprocess(self, r, event, discovery_context): self.discovery_context = discovery_context - description = "contained it" + description = "" tags = [] emit_match = False From bdbd7eefe3182e87dc19468469746dcad065fb07 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 18:21:04 +0000 Subject: [PATCH 011/238] yara docs --- docs/modules/custom_yara_rules.md | 129 ++++++++++++++++++++++++++++-- 1 file changed, 123 insertions(+), 6 deletions(-) diff --git a/docs/modules/custom_yara_rules.md b/docs/modules/custom_yara_rules.md index 087b84be6..6bc9d267c 100644 --- a/docs/modules/custom_yara_rules.md +++ b/docs/modules/custom_yara_rules.md @@ -5,6 +5,57 @@ Though the `excavate` interanal module, BBOT supports searching through HTTP res This feature can be utilized with the command line option `--custom-yara-rules` or `-cy`, followed by a file containing the YARA rules. +Example: + +``` +bbot -m httpx --custom-yara-rules=test.yara -t http://example.com/ +``` + +Where `test.yara` is a file on the filesystem. The file can contain multiple YARA rules, separated by lines. + +YARA rules can be quite simple, the simplest example being a single string search: + +``` +rule find_string { + strings: + $str1 = "AAAABBBB" + + condition: + $str1 +} +``` + +To look for multiple strings, and match if any of them were to hit: + +``` +rule find_string { + strings: + $str1 = "AAAABBBB" + $str2 = "CCCCDDDD" + + condition: + any of them +} +``` + +One of the most important capabilities is the use of regexes within the rule, as shown in the following example. + +``` +rule find_AAAABBBB_regex { + strings: + $regex = /A{1,4}B{1,4}/ + + condition: + $regex +} + +``` + +*Note: YARA uses it's own regex engine that is not a 1:1 match with python regexes. This means many existing regexes will have to be modified before they will work with YARA. The good news is: YARA's regex engine is FAST, immensely more fast than pythons!* + +Further discussion of art of writing complex YARA rules goes far beyond the scope of this documentation. A good place to start learning more is the official YARA [https://yara.readthedocs.io/en/stable/writingrules.html](Writing YARA Rules) documentation. + +The YARA engine provides plenty of room to make highly complex signatures possible, with various conditional operators available. Multiple signatures can be linked together to create sophisticated detection rules that can identify a wide range of specific content. This flexibility allows analysts to craft precise and efficient rules for detecting security vulnerabilities, leveraging logical operators, regular expressions, and other powerful features. Additionally, YARA's modular structure supports easy updates and maintenance of signature sets, making it a robust tool in the arsenal of cybersecurity professionals. ### Custom options @@ -14,20 +65,86 @@ BBOT supports the use of a few custom `meta` attributes within YARA rules, which The description of the rule. Will end up in the description of any produced events if defined. +Example with no description provided: + +``` +[FINDING] {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]", "host": "example.com", "url": "http://example.com"} excavate +``` + +Example with the description added: + +``` +[FINDING] {"description": "Custom Yara Rule [AAAABBBB] with description: [contains our test string] Matched via identifier [str1]", "host": "example.com, "url": "http://example.com"} excavate +``` + +That FINDING was produced with the following signature: + +``` +rule AAAABBBB { + + meta: + description = "contains our test string" + strings: + $str1 = "AAAABBBB" + condition: + $str1 +} +``` + #### tags -Tags specified with this option will be passed-on to any resulting emitted events. Provided as a comma separated string, as shown below: +Tags specified with this option will be passed-on to any resulting emitted events. Tags are rovided as a comma separated string, as shown below: + +Lets expand on the previous example: -TBA +``` +rule AAAABBBB { + + meta: + description = "contains our test string" + tags = "tag1,tag2,tag3" + strings: + $str1 = "AAAABBBB" + condition: + $str1 +} +``` + +Now, the BBOT FINDING includes these custom tags, as with the following output: + +``` +[FINDING] {"description": "Custom Yara Rule [AAAABBBB] with description: [contains our test string] Matched via identifier [str1]", "host": "example.com", "url": "http://example.com/"} excavate (tag1, tag2, tag3) + +``` #### emit_match When set to True, the contents returned from a successful extraction via a YARA regex will be included in the FINDING event which is emitted. -Consider the following example: +Consider the following example YARA rule: + +``` +rule SubstackLink +{ + meta: + description = "contains a Substack link" + emit_match = true + strings: + $substack_link = /https?:\/\/[a-zA-Z0-9.-]+\.substack\.com/ + condition: + $substack_link +} +``` + +When run against the Black Lantern Security homepage with the following BBOT command: + +``` +bbot -m httpx --custom-yara-rules=substack.yara -t http://www.blacklanternsecurity.com/ -TBA +``` -### YARA Resources +We get the following result. Note that the finding now contains the actual link that was identified with the regex. -TBA \ No newline at end of file +``` +[FINDING] {"description": "Custom Yara Rule [SubstackLink] with description: [contains a Substack link] Matched via identifier [substack_link] and extracted [https://blacklanternsecurity.substack.com]", "host": "www.blacklanternsecurity.com", "url": "https://www.blacklanternsecurity.com/"} excavate +``` From ff38ce05b3da2b749659387e5f2288383db5cc96 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 28 Jun 2024 18:39:12 +0000 Subject: [PATCH 012/238] edits --- docs/modules/custom_yara_rules.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/modules/custom_yara_rules.md b/docs/modules/custom_yara_rules.md index 6bc9d267c..9b3e3ec88 100644 --- a/docs/modules/custom_yara_rules.md +++ b/docs/modules/custom_yara_rules.md @@ -1,7 +1,7 @@ # Custom Yara Rules ### Overview -Though the `excavate` interanal module, BBOT supports searching through HTTP response data using custom YARA rules. +Though the `excavate` internal module, BBOT supports searching through HTTP response data using custom YARA rules. This feature can be utilized with the command line option `--custom-yara-rules` or `-cy`, followed by a file containing the YARA rules. @@ -53,9 +53,9 @@ rule find_AAAABBBB_regex { *Note: YARA uses it's own regex engine that is not a 1:1 match with python regexes. This means many existing regexes will have to be modified before they will work with YARA. The good news is: YARA's regex engine is FAST, immensely more fast than pythons!* -Further discussion of art of writing complex YARA rules goes far beyond the scope of this documentation. A good place to start learning more is the official YARA [https://yara.readthedocs.io/en/stable/writingrules.html](Writing YARA Rules) documentation. +Further discussion of art of writing complex YARA rules goes far beyond the scope of this documentation. A good place to start learning more is the [official YARA documentation](https://yara.readthedocs.io/en/stable/writingrules.html). -The YARA engine provides plenty of room to make highly complex signatures possible, with various conditional operators available. Multiple signatures can be linked together to create sophisticated detection rules that can identify a wide range of specific content. This flexibility allows analysts to craft precise and efficient rules for detecting security vulnerabilities, leveraging logical operators, regular expressions, and other powerful features. Additionally, YARA's modular structure supports easy updates and maintenance of signature sets, making it a robust tool in the arsenal of cybersecurity professionals. +The YARA engine provides plenty of room to make highly complex signatures possible, with various conditional operators available. Multiple signatures can be linked together to create sophisticated detection rules that can identify a wide range of specific content. This flexibility allows the crafting of efficient rules for detecting security vulnerabilities, leveraging logical operators, regular expressions, and other powerful features. Additionally, YARA's modular structure supports easy updates and maintenance of signature sets. ### Custom options @@ -93,7 +93,7 @@ rule AAAABBBB { #### tags -Tags specified with this option will be passed-on to any resulting emitted events. Tags are rovided as a comma separated string, as shown below: +Tags specified with this option will be passed-on to any resulting emitted events. Tags are provided as a comma separated string, as shown below: Lets expand on the previous example: @@ -114,7 +114,6 @@ Now, the BBOT FINDING includes these custom tags, as with the following output: ``` [FINDING] {"description": "Custom Yara Rule [AAAABBBB] with description: [contains our test string] Matched via identifier [str1]", "host": "example.com", "url": "http://example.com/"} excavate (tag1, tag2, tag3) - ``` #### emit_match @@ -147,4 +146,4 @@ We get the following result. Note that the finding now contains the actual link ``` [FINDING] {"description": "Custom Yara Rule [SubstackLink] with description: [contains a Substack link] Matched via identifier [substack_link] and extracted [https://blacklanternsecurity.substack.com]", "host": "www.blacklanternsecurity.com", "url": "https://www.blacklanternsecurity.com/"} excavate -``` +``` \ No newline at end of file From 9eadf1f2f719092b8fc6dc5d4a6cff908bc654ea Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:44:39 -0400 Subject: [PATCH 013/238] Update custom_yara_rules.md --- docs/modules/custom_yara_rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/custom_yara_rules.md b/docs/modules/custom_yara_rules.md index 9b3e3ec88..1ec106878 100644 --- a/docs/modules/custom_yara_rules.md +++ b/docs/modules/custom_yara_rules.md @@ -1,7 +1,7 @@ # Custom Yara Rules ### Overview -Though the `excavate` internal module, BBOT supports searching through HTTP response data using custom YARA rules. +Through the `excavate` internal module, BBOT supports searching through HTTP response data using custom YARA rules. This feature can be utilized with the command line option `--custom-yara-rules` or `-cy`, followed by a file containing the YARA rules. @@ -146,4 +146,4 @@ We get the following result. Note that the finding now contains the actual link ``` [FINDING] {"description": "Custom Yara Rule [SubstackLink] with description: [contains a Substack link] Matched via identifier [substack_link] and extracted [https://blacklanternsecurity.substack.com]", "host": "www.blacklanternsecurity.com", "url": "https://www.blacklanternsecurity.com/"} excavate -``` \ No newline at end of file +``` From 51690f1d4dcb37c17e827747bd84ec1089794bfd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 26 Jun 2024 16:08:21 -0400 Subject: [PATCH 014/238] better target tests, whitelist/blacklist improvements --- bbot/scanner/manager.py | 1 + bbot/scanner/scanner.py | 8 +- bbot/scanner/target.py | 106 ++++++++++++++++++++------- bbot/test/test_step_1/test_target.py | 100 ++++++++++++++++++++----- 4 files changed, 163 insertions(+), 52 deletions(-) diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index a1ced01fd..2ae574fa5 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -54,6 +54,7 @@ async def init_events(self, events=None): event.parent = self.scan.root_event if event.module is None: event.module = self.scan._make_dummy_module(name="TARGET", _type="TARGET") + event.add_tag("target") if event != self.scan.root_event: event.discovery_context = f"Scan {self.scan.name} seeded with " + "{event.type}: {event.data}" self.verbose(f"Target: {event}") diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 724e86872..7493f9ea8 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -998,12 +998,8 @@ def json(self): v = getattr(self, i, "") if v: j.update({i: v}) - j["target_hash"] = self.target.hash.hex() - j["seed_hash"] = self.target.seeds.hash.hex() - j["whitelist_hash"] = self.target.whitelist.hash.hex() - j["blacklist_hash"] = self.target.blacklist.hash.hex() - j["scope_hash"] = self.target.scope_hash.hex() - j["preset"] = self.preset.to_dict(include_target=True, redact_secrets=True) + j["target"] = self.preset.target.json + j["preset"] = self.preset.to_dict(redact_secrets=True) return j def debug(self, *args, trace=False, **kwargs): diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 55fd509f5..dea03c4dd 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -32,12 +32,12 @@ def __init__(self, *targets, whitelist=None, blacklist=None, strict_scope=False, whitelist = set([e.host for e in self.seeds if e.host]) else: log.verbose(f"Creating events from {len(whitelist):,} whitelist entries") - self.whitelist = Target(*whitelist, strict_scope=self.strict_scope, scan=scan, hosts_only=True) + self.whitelist = Target(*whitelist, strict_scope=self.strict_scope, scan=scan, acl_mode=True) if blacklist is None: blacklist = [] if blacklist: log.verbose(f"Creating events from {len(blacklist):,} blacklist entries") - self.blacklist = Target(*blacklist, scan=scan, hosts_only=True) + self.blacklist = Target(*blacklist, scan=scan, acl_mode=True) self._hash = None def add(self, *args, **kwargs): @@ -108,6 +108,20 @@ def scope_hash(self): sha1_hash.update(target_hash) return sha1_hash.digest() + @property + def json(self): + return { + "seeds": sorted([e.data for e in self.seeds]), + "whitelist": sorted([e.data for e in self.whitelist]), + "blacklist": sorted([e.data for e in self.blacklist]), + "strict_scope": self.strict_scope, + "hash": self.hash.hex(), + "seed_hash": self.seeds.hash.hex(), + "whitelist_hash": self.whitelist.hash.hex(), + "blacklist_hash": self.blacklist.hash.hex(), + "scope_hash": self.scope_hash.hex(), + } + def copy(self): self_copy = copy.copy(self) self_copy.seeds = self.seeds.copy() @@ -244,7 +258,7 @@ class Target: - If you do not want to include child subdomains, use `strict_scope=True` """ - def __init__(self, *targets, strict_scope=False, scan=None, hosts_only=False): + def __init__(self, *targets, strict_scope=False, scan=None, acl_mode=False): """ Initialize a Target object. @@ -252,7 +266,7 @@ def __init__(self, *targets, strict_scope=False, scan=None, hosts_only=False): *targets: One or more targets (e.g., domain names, IP ranges) to be included in this Target. strict_scope (bool): Whether to consider subdomains of target domains in-scope scan (Scan): Reference to the Scan object that instantiated the Target. - hosts_only (bool): Whether to discard events and keep only their hosts + acl_mode (bool): Stricter deduplication for more efficient checks Notes: - If you are instantiating a target from within a BBOT module, use `self.helpers.make_target()` instead. (this removes the need to pass in a scan object.) @@ -261,7 +275,7 @@ def __init__(self, *targets, strict_scope=False, scan=None, hosts_only=False): """ self.scan = scan self.strict_scope = strict_scope - self.hosts_only = hosts_only + self.acl_mode = acl_mode self.special_event_types = { "ORG_STUB": re.compile(r"^ORG:(.*)", re.IGNORECASE), "ASN": re.compile(r"^ASN:(.*)", re.IGNORECASE), @@ -269,10 +283,8 @@ def __init__(self, *targets, strict_scope=False, scan=None, hosts_only=False): self._events = set() self._radix = RadixTarget() - for t in targets: - if t == "": - assert False - self.add(t) + for target_event in self._make_events(targets): + self._add_event(target_event) self._hash = None @@ -303,13 +315,6 @@ def add(self, t, event_type=None): if is_event(single_target): event = single_target else: - single_target = str(single_target) - for eventtype, regex in self.special_event_types.items(): - match = regex.match(single_target) - if match: - single_target = match.groups()[0] - event_type = eventtype - break try: event = make_event( single_target, event_type=event_type, dummy=True, tags=["target"], scan=self.scan @@ -339,6 +344,10 @@ def events(self): """ return self._events + @property + def hosts(self): + return [e.host for e in self.events] + def copy(self): """ Creates and returns a copy of the Target object, including a shallow copy of the `_events` and `_radix` attributes. @@ -412,20 +421,63 @@ def get_host(self, host): return return event + def _len_event(self, event): + """ + Used for sorting events by their length, so that bigger ones (i.e. ) + """ + try: + # smaller domains should come first + return len(event.host) + except TypeError: + try: + # bigger IP subnets should come first + return -event.host.num_addresses + except AttributeError: + # IP addresses default to 1 + return 1 + + def _sort_events(self, events): + return sorted(events, key=self._len_event) + + def _make_events(self, targets): + events = [] + for target in targets: + event_type = None + for eventtype, regex in self.special_event_types.items(): + if isinstance(target, str): + match = regex.match(target) + if match: + target = match.groups()[0] + event_type = eventtype + break + events.append(make_event(target, event_type=event_type, dummy=True, scan=self.scan)) + return self._sort_events(events) + def _add_event(self, event): + skip = False if event.host: radix_data = self._radix.search(event.host) - if self.hosts_only: - event_type = "IP_RANGE" if event.type == "IP_RANGE" else "DNS_NAME" - event = make_event(event.host, event_type=event_type, dummy=True, tags=["target"], scan=self.scan) - if radix_data is None: - radix_data = {event} - self._radix.insert(event.host, radix_data) - else: - radix_data.add(event) - # clear hash - self._hash = None - self._events.add(event) + if self.acl_mode: + # skip if the hostname/IP/subnet (or its parent) has already been added + if radix_data is not None: + skip = True + else: + event_type = "IP_RANGE" if event.type == "IP_RANGE" else "DNS_NAME" + event = make_event(event.host, event_type=event_type, dummy=True, scan=self.scan) + if not skip: + if radix_data is None: + radix_data = {event} + self._radix.insert(event.host, radix_data) + else: + radix_data.add(event) + # clear hash + self._hash = None + else: + # skip if we're in ACL mode and there's no host + if self.acl_mode: + skip = True + if not skip: + self._events.add(event) def _contains(self, other): if self.get(other) is not None: diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 562455e2f..253d63eb8 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -3,6 +3,10 @@ @pytest.mark.asyncio async def test_target(bbot_scanner): + import random + from ipaddress import ip_address, ip_network + from bbot.scanner.target import Target, BBOTTarget + scan1 = bbot_scanner("api.publicapis.org", "8.8.8.8/30", "2001:4860:4860::8888/126") scan2 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125") scan3 = bbot_scanner("8.8.8.8/29", "publicapis.org", "2001:4860:4860::8888/125") @@ -73,8 +77,6 @@ async def test_target(bbot_scanner): assert str(scan1.target.get("www.api.publicapis.org").host) == "api.publicapis.org" assert scan1.target.get("publicapis.org") is None - from bbot.scanner.target import Target, BBOTTarget - target = Target("evilcorp.com") assert not "com" in target assert "evilcorp.com" in target @@ -189,10 +191,20 @@ async def test_target(bbot_scanner): whitelist=["evilcorp.com", "bob@www.evilcorp.com", "evilcorp.net"], blacklist=["1.2.3.4", "4.3.2.1/24", "http://1.2.3.4", "bob@asdf.evilcorp.net"], ) - assert bbottarget.hash == b"\x8dW\xcbA\x0c\xc5\r\xc0\xfa\xae\xcd\xfc\x8e[<\xb5\x06\xc87\xf9" - assert bbottarget.scope_hash == b"/\xce\xbf\x013\xb2\xb8\xf6\xbe_@\xae\xfc\x17w]\x85\x15N9" + assert set([e.data for e in bbottarget.seeds.events]) == { + "1.2.3.0/24", + "http://www.evilcorp.net/", + "bob@fdsa.evilcorp.net", + } + assert set([e.data for e in bbottarget.whitelist.events]) == {"evilcorp.com", "evilcorp.net"} + assert set([e.data for e in bbottarget.blacklist.events]) == {"1.2.3.4", "4.3.2.0/24", "asdf.evilcorp.net"} + assert set(bbottarget.seeds.hosts) == {ip_network("1.2.3.0/24"), "www.evilcorp.net", "fdsa.evilcorp.net"} + assert set(bbottarget.whitelist.hosts) == {"evilcorp.com", "evilcorp.net"} + assert set(bbottarget.blacklist.hosts) == {ip_address("1.2.3.4"), ip_network("4.3.2.0/24"), "asdf.evilcorp.net"} + assert bbottarget.hash == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba" + assert bbottarget.scope_hash == b"\x00\xf5V\xfb.\xeb#\xcb\xf0q\xf9\xe9e\xb7\x1f\xe2T+\xdbw" assert bbottarget.seeds.hash == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5" - assert bbottarget.whitelist.hash == b"b\x95\xc5\xf0hQ\x0c\x08\x92}\xa55\xff\x83\xf9'\x93\x927\xcb" + assert bbottarget.whitelist.hash == b"\xa0Af\x07n\x10\xd9\xb6\n\xa7TO\xb07\xcdW\xc4vLC" assert bbottarget.blacklist.hash == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/" scan = bbot_scanner( @@ -205,17 +217,67 @@ async def test_target(bbot_scanner): events = [e async for e in scan.async_start()] scan_events = [e for e in events if e.type == "SCAN"] assert len(scan_events) == 1 - assert ( - scan_events[0].data["target_hash"] == b"\x8dW\xcbA\x0c\xc5\r\xc0\xfa\xae\xcd\xfc\x8e[<\xb5\x06\xc87\xf9".hex() - ) - assert scan_events[0].data["scope_hash"] == b"/\xce\xbf\x013\xb2\xb8\xf6\xbe_@\xae\xfc\x17w]\x85\x15N9".hex() - assert scan_events[0].data["seed_hash"] == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5".hex() - assert ( - scan_events[0].data["whitelist_hash"] == b"b\x95\xc5\xf0hQ\x0c\x08\x92}\xa55\xff\x83\xf9'\x93\x927\xcb".hex() - ) - assert scan_events[0].data["blacklist_hash"] == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/".hex() - assert scan_events[0].data["target_hash"] == "8d57cb410cc50dc0faaecdfc8e5b3cb506c837f9" - assert scan_events[0].data["scope_hash"] == "2fcebf0133b2b8f6be5f40aefc17775d85154e39" - assert scan_events[0].data["seed_hash"] == "af2e8683a143adb4e7605894e2a001c2e33a4ac5" - assert scan_events[0].data["whitelist_hash"] == "6295c5f068510c08927da535ff83f927939237cb" - assert scan_events[0].data["blacklist_hash"] == "af0e8ae94a5a86beeea9a9db30af27238420552f" + target_dict = scan_events[0].data["target"] + assert target_dict["strict_scope"] == False + assert target_dict["hash"] == b"\x0b\x908\xe3\xef\n=\x13d\xdf\x00;\xack\x0c\xbc\xd2\xcc'\xba".hex() + assert target_dict["scope_hash"] == b"\x00\xf5V\xfb.\xeb#\xcb\xf0q\xf9\xe9e\xb7\x1f\xe2T+\xdbw".hex() + assert target_dict["seed_hash"] == b"\xaf.\x86\x83\xa1C\xad\xb4\xe7`X\x94\xe2\xa0\x01\xc2\xe3:J\xc5".hex() + assert target_dict["whitelist_hash"] == b"\xa0Af\x07n\x10\xd9\xb6\n\xa7TO\xb07\xcdW\xc4vLC".hex() + assert target_dict["blacklist_hash"] == b"\xaf\x0e\x8a\xe9JZ\x86\xbe\xee\xa9\xa9\xdb0\xaf'#\x84 U/".hex() + assert target_dict["hash"] == "0b9038e3ef0a3d1364df003bac6b0cbcd2cc27ba" + assert target_dict["scope_hash"] == "00f556fb2eeb23cbf071f9e965b71fe2542bdb77" + assert target_dict["seed_hash"] == "af2e8683a143adb4e7605894e2a001c2e33a4ac5" + assert target_dict["whitelist_hash"] == "a04166076e10d9b60aa7544fb037cd57c4764c43" + assert target_dict["blacklist_hash"] == "af0e8ae94a5a86beeea9a9db30af27238420552f" + + # test target sorting + big_subnet = scan.make_event("1.2.3.4/24", dummy=True) + medium_subnet = scan.make_event("1.2.3.4/28", dummy=True) + small_subnet = scan.make_event("1.2.3.4/30", dummy=True) + ip_event = scan.make_event("1.2.3.4", dummy=True) + parent_domain = scan.make_event("evilcorp.com", dummy=True) + grandparent_domain = scan.make_event("www.evilcorp.com", dummy=True) + greatgrandparent_domain = scan.make_event("api.www.evilcorp.com", dummy=True) + target = Target() + assert target._len_event(big_subnet) == -256 + assert target._len_event(medium_subnet) == -16 + assert target._len_event(small_subnet) == -4 + assert target._len_event(ip_event) == 1 + assert target._len_event(parent_domain) == 12 + assert target._len_event(grandparent_domain) == 16 + assert target._len_event(greatgrandparent_domain) == 20 + events = [ + big_subnet, + medium_subnet, + small_subnet, + ip_event, + parent_domain, + grandparent_domain, + greatgrandparent_domain, + ] + random.shuffle(events) + assert target._sort_events(events) == [ + big_subnet, + medium_subnet, + small_subnet, + ip_event, + parent_domain, + grandparent_domain, + greatgrandparent_domain, + ] + + # make sure child subnets/IPs don't get added to whitelist/blacklist + target = Target("1.2.3.4/24", "1.2.3.4/28", acl_mode=True) + assert set(e.data for e in target) == {"1.2.3.0/24"} + target = Target("1.2.3.4/28", "1.2.3.4/24", acl_mode=True) + assert set(e.data for e in target) == {"1.2.3.0/24"} + target = Target("1.2.3.4/28", "1.2.3.4", acl_mode=True) + assert set(e.data for e in target) == {"1.2.3.0/28"} + target = Target("1.2.3.4", "1.2.3.4/28", acl_mode=True) + assert set(e.data for e in target) == {"1.2.3.0/28"} + + # same but for domains + target = Target("evilcorp.com", "www.evilcorp.com", acl_mode=True) + assert set(e.data for e in target) == {"evilcorp.com"} + target = Target("www.evilcorp.com", "evilcorp.com", acl_mode=True) + assert set(e.data for e in target) == {"evilcorp.com"} From 190f51d091fa41e70a542c94a7e7879d1d919a09 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 26 Jun 2024 17:02:20 -0400 Subject: [PATCH 015/238] steady work on bbot-io compatibility --- bbot/scanner/target.py | 13 +++++++------ bbot/test/test_step_1/test_target.py | 8 ++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index dea03c4dd..23e68e5f4 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -423,7 +423,7 @@ def get_host(self, host): def _len_event(self, event): """ - Used for sorting events by their length, so that bigger ones (i.e. ) + Used for sorting events by their length, so that bigger ones (e.g. IP subnets) are added first """ try: # smaller domains should come first @@ -459,23 +459,24 @@ def _add_event(self, event): radix_data = self._radix.search(event.host) if self.acl_mode: # skip if the hostname/IP/subnet (or its parent) has already been added - if radix_data is not None: + if radix_data is not None and not self.strict_scope: skip = True else: event_type = "IP_RANGE" if event.type == "IP_RANGE" else "DNS_NAME" event = make_event(event.host, event_type=event_type, dummy=True, scan=self.scan) if not skip: - if radix_data is None: + # if strict scope is enabled and it's not an exact host match, we add a whole new entry + if radix_data is None or (self.strict_scope and event.host not in radix_data): radix_data = {event} self._radix.insert(event.host, radix_data) + # otherwise, we add the event to the set else: radix_data.add(event) # clear hash self._hash = None - else: + elif self.acl_mode and not self.strict_scope: # skip if we're in ACL mode and there's no host - if self.acl_mode: - skip = True + skip = True if not skip: self._events.add(event) diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 253d63eb8..402ac77c2 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -281,3 +281,11 @@ async def test_target(bbot_scanner): assert set(e.data for e in target) == {"evilcorp.com"} target = Target("www.evilcorp.com", "evilcorp.com", acl_mode=True) assert set(e.data for e in target) == {"evilcorp.com"} + + # make sure strict_scope doesn't mess us up + target = Target("evilcorp.co.uk", "www.evilcorp.co.uk", acl_mode=True, strict_scope=True) + assert set(target.hosts) == {"evilcorp.co.uk", "www.evilcorp.co.uk"} + assert "evilcorp.co.uk" in target + assert "www.evilcorp.co.uk" in target + assert not "api.evilcorp.co.uk" in target + assert not "api.www.evilcorp.co.uk" in target From 400ef1308c7d75769b5e2a6d56aa78bc2db5871e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 28 Jun 2024 16:25:59 -0400 Subject: [PATCH 016/238] fix scan tests --- bbot/test/test_step_1/test_scan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index 3e5660abe..71e80e1ee 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -9,6 +9,7 @@ async def test_scan( bbot_scanner, ): scan0 = bbot_scanner( + "1.1.1.0", "1.1.1.1/31", "evilcorp.com", blacklist=["1.1.1.1/28", "www.evilcorp.com"], @@ -30,9 +31,9 @@ async def test_scan( assert not scan0.in_scope("test.www.evilcorp.com") assert not scan0.in_scope("www.evilcorp.co.uk") j = scan0.json - assert "1.1.1.0/31" in j["preset"]["target"] - assert "whitelist" not in j["preset"] - assert "1.1.1.0/28" in j["preset"]["blacklist"] + assert set(j["target"]["seeds"]) == {"1.1.1.0", "1.1.1.0/31", "evilcorp.com"} + assert set(j["target"]["whitelist"]) == {"1.1.1.0/31", "evilcorp.com"} + assert set(j["target"]["blacklist"]) == {"1.1.1.0/28", "www.evilcorp.com"} assert "ipneighbor" in j["preset"]["modules"] scan1 = bbot_scanner("1.1.1.1", whitelist=["1.0.0.1"]) From 54fac7fa06423e694e55f1f5aef2e35f2b42c243 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 09:37:34 -0400 Subject: [PATCH 017/238] fix json tests --- bbot/test/test_step_2/module_tests/test_module_json.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 841a777f9..6a5215f6e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -22,7 +22,8 @@ def check(self, module_test, events): dns_json = dns_json[0] assert scan_json["data"]["name"] == module_test.scan.name assert scan_json["data"]["id"] == module_test.scan.id - assert scan_json["data"]["preset"]["target"] == ["blacklanternsecurity.com"] + assert scan_json["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["discovery_context"] == context_data assert dns_json["discovery_path"] == [context_data] @@ -32,7 +33,8 @@ def check(self, module_test, events): dns_reconstructed = event_from_json(dns_json) assert scan_reconstructed.data["name"] == module_test.scan.name assert scan_reconstructed.data["id"] == module_test.scan.id - assert scan_reconstructed.data["preset"]["target"] == ["blacklanternsecurity.com"] + assert scan_reconstructed.data["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_reconstructed.data == dns_data assert dns_reconstructed.discovery_context == context_data assert dns_reconstructed.discovery_path == [context_data] From 037c625038e12c61b5c0a2c5fba13574fb8de7af Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 09:58:48 -0400 Subject: [PATCH 018/238] cleaner shutdown of engines --- bbot/core/core.py | 1 + bbot/core/engine.py | 20 ++++++++++++++++++++ bbot/scanner/scanner.py | 6 ++++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index 1a51afd1a..ccb838ee4 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -184,6 +184,7 @@ def create_process(self, *args, **kwargs): import threading kwargs.pop("custom_name", None) + kwargs["daemon"] = True process = threading.Thread(*args, **kwargs) else: from .helpers.process import BBOTProcess diff --git a/bbot/core/engine.py b/bbot/core/engine.py index f9b601491..4fbfc62cd 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -51,6 +51,7 @@ class EngineClient(EngineBase): SERVER_CLASS = None def __init__(self, **kwargs): + self._shutdown = False super().__init__() self.name = f"EngineClient {self.__class__.__name__}" self.process = None @@ -76,6 +77,9 @@ def check_error(self, message): return False async def run_and_return(self, command, *args, **kwargs): + if self._shutdown: + self.log.verbose("Engine has been shut down and is not accepting new tasks") + return async with self.new_socket() as socket: try: message = self.make_message(command, args=args, kwargs=kwargs) @@ -97,6 +101,9 @@ async def run_and_return(self, command, *args, **kwargs): return message async def run_and_yield(self, command, *args, **kwargs): + if self._shutdown: + self.log.verbose("Engine has been shut down and is not accepting new tasks") + return message = self.make_message(command, args=args, kwargs=kwargs) if message is error_sentinel: return @@ -182,6 +189,14 @@ async def new_socket(self): with suppress(Exception): socket.close() + async def shutdown(self): + self._shutdown = True + async with self.new_socket() as socket: + # -99 == special shutdown signal + shutdown_message = pickle.dumps({"c": -99}) + await socket.send(shutdown_message) + self.cleanup() + def cleanup(self): # delete socket file on exit self.socket_path.unlink(missing_ok=True) @@ -276,6 +291,7 @@ async def worker(self): self.log.warning(f"No command sent in message: {message}") continue + # -1 == cancel task if cmd == -1: task = self.tasks.get(client_id, None) if task is None: @@ -293,6 +309,10 @@ async def worker(self): self.tasks.pop(client_id, None) continue + # -99 == shut down engine + if cmd == -99: + break + args = message.get("a", ()) if not isinstance(args, tuple): self.log.warning(f"{self.name}: received invalid args of type {type(args)}, should be tuple") diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 7493f9ea8..498e1550d 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -786,8 +786,10 @@ async def _cleanup(self): None """ self.status = "CLEANING_UP" - # clean up dns engine - self.helpers.dns.cleanup() + # shut down dns engine + await self.helpers.dns.shutdown() + # shut down web engine + await self.helpers.web.shutdown() # clean up modules for mod in self.modules.values(): await mod._cleanup() From e81115cf0c73ec2f6f16765b8989fea11be785aa Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 17:49:44 -0400 Subject: [PATCH 019/238] small engine tweak --- bbot/core/core.py | 9 +++++++-- bbot/core/engine.py | 28 +++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index ccb838ee4..9a742856c 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -5,6 +5,8 @@ from contextlib import suppress from omegaconf import OmegaConf +from bbot.errors import BBOTError + DEFAULT_CONFIG = None @@ -187,9 +189,12 @@ def create_process(self, *args, **kwargs): kwargs["daemon"] = True process = threading.Thread(*args, **kwargs) else: - from .helpers.process import BBOTProcess + if self.process_name == "MainProcess": + from .helpers.process import BBOTProcess - process = BBOTProcess(*args, **kwargs) + process = BBOTProcess(*args, **kwargs) + else: + raise BBOTError(f"Tried to start server from process {self.process_name}") process.daemon = True return process diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 4fbfc62cd..02bd2af7e 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -147,20 +147,17 @@ def available_commands(self): return [s for s in self.CMDS if isinstance(s, str)] def start_server(self): - if self.process_name == "MainProcess": - self.process = CORE.create_process( - target=self.server_process, - args=( - self.SERVER_CLASS, - self.socket_path, - ), - kwargs=self.server_kwargs, - custom_name="bbot dnshelper", - ) - self.process.start() - return self.process - else: - raise BBOTEngineError(f"Tried to start server from process {self.process_name}") + self.process = CORE.create_process( + target=self.server_process, + args=( + self.SERVER_CLASS, + self.socket_path, + ), + kwargs=self.server_kwargs, + custom_name="bbot dnshelper", + ) + self.process.start() + return self.process @staticmethod def server_process(server_class, socket_path, **kwargs): @@ -311,7 +308,8 @@ async def worker(self): # -99 == shut down engine if cmd == -99: - break + self.log.verbose("Got shutdown signal, shutting down...") + return args = message.get("a", ()) if not isinstance(args, tuple): From f8668d26a1df3f68efe1329bbf76480804d8feb4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 17:53:35 -0400 Subject: [PATCH 020/238] pytest things --- bbot/core/core.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index 9a742856c..b382a999e 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -185,9 +185,12 @@ def create_process(self, *args, **kwargs): if os.environ.get("BBOT_TESTING", "") == "True": import threading - kwargs.pop("custom_name", None) - kwargs["daemon"] = True - process = threading.Thread(*args, **kwargs) + if threading.current_thread() is threading.main_thread(): + kwargs.pop("custom_name", None) + kwargs["daemon"] = True + process = threading.Thread(*args, **kwargs) + else: + raise BBOTError(f"Tried to start server from process {self.process_name}") else: if self.process_name == "MainProcess": from .helpers.process import BBOTProcess From 68aaa2766c02a81da496068f038eb6f33e3c21ed Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 18:10:10 -0400 Subject: [PATCH 021/238] engine tweaks --- bbot/core/core.py | 29 +++++++++++++++++++++++------ bbot/core/engine.py | 2 -- bbot/core/helpers/process.py | 1 + 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index b382a999e..d34374a21 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -43,6 +43,10 @@ def __init__(self): self.logger self.log = logging.getLogger("bbot.core") + import multiprocessing + + self.process_name = multiprocessing.current_process().name + @property def home(self): return Path(self.config["home"]).expanduser().resolve() @@ -185,12 +189,25 @@ def create_process(self, *args, **kwargs): if os.environ.get("BBOT_TESTING", "") == "True": import threading - if threading.current_thread() is threading.main_thread(): - kwargs.pop("custom_name", None) - kwargs["daemon"] = True - process = threading.Thread(*args, **kwargs) - else: - raise BBOTError(f"Tried to start server from process {self.process_name}") + class BBOTThread(threading.Thread): + + default_name = "default bbot thread" + + def __init__(_self, *args, **kwargs): + _self.custom_name = kwargs.pop("custom_name", _self.default_name) + super().__init__(*args, **kwargs) + + def run(_self): + from setproctitle import setproctitle + + setproctitle(str(_self.custom_name)) + super().run() + + # if threading.current_thread() is threading.main_thread(): + kwargs.pop("custom_name", None) + process = BBOTThread(*args, **kwargs) + # else: + # raise BBOTError(f"Tried to start server from process {self.process_name}") else: if self.process_name == "MainProcess": from .helpers.process import BBOTProcess diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 02bd2af7e..8c7d87016 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -7,7 +7,6 @@ import tempfile import traceback import zmq.asyncio -import multiprocessing from pathlib import Path from contextlib import asynccontextmanager, suppress @@ -55,7 +54,6 @@ def __init__(self, **kwargs): super().__init__() self.name = f"EngineClient {self.__class__.__name__}" self.process = None - self.process_name = multiprocessing.current_process().name if self.SERVER_CLASS is None: raise ValueError(f"Must set EngineClient SERVER_CLASS, {self.SERVER_CLASS}") self.CMDS = dict(self.SERVER_CLASS.CMDS) diff --git a/bbot/core/helpers/process.py b/bbot/core/helpers/process.py index 90843b441..0243cfc9a 100644 --- a/bbot/core/helpers/process.py +++ b/bbot/core/helpers/process.py @@ -50,5 +50,6 @@ def run(self): BBOTProcess.__init__, log_level=CORE.logger.log_level, log_queue=CORE.logger.queue ) +# this makes our process class the default for process pools, etc. mp_context = multiprocessing.get_context("spawn") mp_context.Process = BBOTProcess From 0774092377caabb13e7bbc713d2f98fe0ecbf6a9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 19:55:48 -0400 Subject: [PATCH 022/238] pytest tweaks --- bbot/core/core.py | 24 ++++++------------------ bbot/core/engine.py | 2 -- bbot/core/helpers/async_helpers.py | 5 +++-- bbot/core/helpers/files.py | 13 ++++++++++--- bbot/core/helpers/process.py | 16 ++++++++++++++++ bbot/test/bbot_fixtures.py | 2 +- 6 files changed, 36 insertions(+), 26 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index d34374a21..05a8af297 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -187,25 +187,8 @@ def files_config(self): def create_process(self, *args, **kwargs): if os.environ.get("BBOT_TESTING", "") == "True": - import threading - - class BBOTThread(threading.Thread): - - default_name = "default bbot thread" - - def __init__(_self, *args, **kwargs): - _self.custom_name = kwargs.pop("custom_name", _self.default_name) - super().__init__(*args, **kwargs) - - def run(_self): - from setproctitle import setproctitle - - setproctitle(str(_self.custom_name)) - super().run() - # if threading.current_thread() is threading.main_thread(): - kwargs.pop("custom_name", None) - process = BBOTThread(*args, **kwargs) + process = self.create_thread(*args, **kwargs) # else: # raise BBOTError(f"Tried to start server from process {self.process_name}") else: @@ -218,6 +201,11 @@ def run(_self): process.daemon = True return process + def create_thread(self, *args, **kwargs): + from .helpers.process import BBOTThread + + return BBOTThread(*args, **kwargs) + @property def logger(self): self.config diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 8c7d87016..1674a7e6b 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -14,8 +14,6 @@ from bbot.errors import BBOTEngineError from bbot.core.helpers.misc import rand_string -CMD_EXIT = 1000 - error_sentinel = object() diff --git a/bbot/core/helpers/async_helpers.py b/bbot/core/helpers/async_helpers.py index 8434ccb0f..dcc510ee4 100644 --- a/bbot/core/helpers/async_helpers.py +++ b/bbot/core/helpers/async_helpers.py @@ -2,7 +2,6 @@ import random import asyncio import logging -import threading from datetime import datetime from queue import Queue, Empty from cachetools import LRUCache @@ -118,8 +117,10 @@ def generator(): if is_done: break + from .process import BBOTThread + # Start the event loop in a separate thread - thread = threading.Thread(target=lambda: asyncio.run(runner())) + thread = BBOTThread(target=lambda: asyncio.run(runner()), daemon=True, custom_name="bbot async_to_sync_gen()") thread.start() # Return the generator diff --git a/bbot/core/helpers/files.py b/bbot/core/helpers/files.py index 438f74112..fb92d1c8b 100644 --- a/bbot/core/helpers/files.py +++ b/bbot/core/helpers/files.py @@ -1,6 +1,5 @@ import os import logging -import threading import traceback from contextlib import suppress @@ -104,7 +103,13 @@ def feed_pipe(self, pipe, content, text=True): text (bool, optional): If True, the content is decoded using smart_decode function. If False, smart_encode function is used. Defaults to True. """ - t = threading.Thread(target=self._feed_pipe, args=(pipe, content), kwargs={"text": text}, daemon=True) + t = self.preset.core.create_thread( + target=self._feed_pipe, + args=(pipe, content), + kwargs={"text": text}, + daemon=True, + custom_name="bbot feed_pipe()", + ) t.start() @@ -127,7 +132,9 @@ def tempfile_tail(self, callback): rm_at_exit(filename) try: os.mkfifo(filename) - t = threading.Thread(target=tail, args=(filename, callback), daemon=True) + t = self.preset.core.create_thread( + target=tail, args=(filename, callback), daemon=True, custom_name="bbot tempfile_tail()" + ) t.start() except Exception as e: log.error(f"Error setting up tail for file {filename}: {e}") diff --git a/bbot/core/helpers/process.py b/bbot/core/helpers/process.py index 0243cfc9a..a00a93994 100644 --- a/bbot/core/helpers/process.py +++ b/bbot/core/helpers/process.py @@ -1,5 +1,6 @@ import logging import traceback +import threading import multiprocessing from multiprocessing.context import SpawnProcess @@ -9,6 +10,21 @@ current_process = multiprocessing.current_process() +class BBOTThread(threading.Thread): + + default_name = "default bbot thread" + + def __init__(self, *args, **kwargs): + self.custom_name = kwargs.pop("custom_name", self.default_name) + super().__init__(*args, **kwargs) + + def run(self): + from setproctitle import setproctitle + + setproctitle(str(self.custom_name)) + super().run() + + class BBOTProcess(SpawnProcess): default_name = "bbot process pool" diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 309edcafc..e60da6fb8 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -203,7 +203,7 @@ class bbot_events: return bbot_events -@pytest.fixture(autouse=True) +@pytest.fixture(scope="session", autouse=True) def install_all_python_deps(): deps_pip = set() for module in DEFAULT_PRESET.module_loader.preloaded().values(): From d944b0413d49e3852b5cbd1db264f93d6c788662 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 28 Jun 2024 18:15:23 -0400 Subject: [PATCH 023/238] portscan improvements --- bbot/modules/portscan.py | 244 ++++++++++++++++++++++----------------- 1 file changed, 138 insertions(+), 106 deletions(-) diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index 79d72c63e..7a5f925da 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -50,6 +50,7 @@ async def setup(self): self.wait = self.config.get("wait", 10) self.ping_first = self.config.get("ping_first", False) self.ping_only = self.config.get("ping_only", False) + self.ping_scan = self.ping_first or self.ping_only self.adapter = self.config.get("adapter", "") self.adapter_ip = self.config.get("adapter_ip", "") self.adapter_mac = self.config.get("adapter_mac", "") @@ -60,8 +61,13 @@ async def setup(self): self.helpers.parse_port_string(self.ports) except ValueError as e: return False, f"Error parsing ports: {e}" - self.alive_hosts = dict() - self.scanned_tracker = RadixTarget() + # keeps track of individual scanned IPs and their open ports + # this is necessary because we may encounter more hosts with the same IP + # and we want to avoid scanning them again + self.open_ports_cache = {} + self.alive_hosts_cache = {} + # keeps track of which IPs/subnets have already been scanned + self.scanned_targets = RadixTarget() self.prep_blacklist() self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges") # check if we're set up for IPv6 @@ -81,48 +87,25 @@ async def setup(self): return True async def handle_batch(self, *events): - targets = [str(h) for h in self.make_targets(events)] - # ping scan - if self.ping_first or self.ping_only: - new_targets = [] - async for alive_host, _ in self.masscan(targets, ping=True): - parent_event = self.scanned_tracker.search(alive_host) - # masscan gets the occasional junk result - # this seems to be a side effect of it having its own TCP stack - # see https://github.com/robertdavidgraham/masscan/issues/397 - if parent_event is None: - self.debug(f"Failed to correlate {alive_host} to targets") - continue - await self.emit_event( - alive_host, - "DNS_NAME", - parent=parent_event, - context=f"{{module}} pinged {parent_event.data} and got a response: {{event.type}}: {{event.data}}", - ) - new_targets.append(ipaddress.ip_network(alive_host, strict=False)) - targets = new_targets + if self.ping_scan: + ping_targets, ping_correlator = await self.make_targets(events, self.alive_hosts_cache) + syn_targets, syn_correlator = RadixTarget(self.open_ports_cache) + async for alive_host, _, parent_event in self.masscan(ping_targets, ping_correlator, ping=True): + # port 0 means icmp ping response + await self.emit_open_port(alive_host, 0, parent_event) + syn_targets.insert(ipaddress.ip_network(alive_host, strict=False), parent_event) + else: + syn_targets, syn_correlator = await self.make_targets(events, self.open_ports_cache) # TCP SYN scan if not self.ping_only: - async for host, port in self.masscan(targets): - parent_event = self.scanned_tracker.search(host) - if parent_event is None: - self.debug(f"Failed to correlate {host} to targets") - continue - if parent_event.type == "DNS_NAME": - host = parent_event.host - netloc = self.helpers.make_netloc(host, port) - await self.emit_event( - netloc, - "OPEN_TCP_PORT", - parent=parent_event, - context=f"{{module}} executed a TCP SYN scan against {parent_event.data} and found: {{event.type}}: {{event.data}}", - ) + async for ip, port, parent_event in self.masscan(syn_targets, syn_correlator): + await self.emit_open_port(ip, port, parent_event) else: self.verbose("Only ping sweep was requested, skipping TCP SYN scan") - async def masscan(self, targets, ping=False): + async def masscan(self, targets, correlator, ping=False): scan_type = "ping" if ping else "SYN" self.verbose(f"Starting masscan {scan_type} scan") if not targets: @@ -135,90 +118,84 @@ async def masscan(self, targets, ping=False): try: with open(stats_file, "w") as stats_fh: async for line in self.run_process_live(command, sudo=True, stderr=stats_fh): - for host, port in self.parse_json_line(line): - yield host, port + for ip, port in self.parse_json_line(line): + parent_event = correlator.search(ip) + # masscan gets the occasional junk result. this is harmless and + # seems to be a side effect of it having its own TCP stack + # see https://github.com/robertdavidgraham/masscan/issues/397 + if parent_event is None: + self.debug(f"Failed to correlate {ip} to targets") + continue + if parent_event.type == "DNS_NAME": + host = parent_event.host + else: + host = ip + yield host, port, parent_event finally: for file in (stats_file, target_file): file.unlink() - def log_masscan_status(self, s): - if "FAIL" in s: - self.warning(s) - self.warning( - f'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.' - ) - else: - self.verbose(s) - - def _build_masscan_command(self, target_file=None, ping=False, dry_run=False, wait=None): - if wait is None: - wait = self.wait - command = ( - "masscan", - "--excludefile", - str(self.exclude_file), - "--rate", - self.rate, - "--wait", - wait, - "--open-only", - "-oJ", - "-", - ) - if target_file is not None: - command += ("-iL", str(target_file)) - if dry_run: - command += ("-p1", "--wait", "0") - else: - if self.adapter: - command += ("--adapter", self.adapter) - if self.adapter_ip: - command += ("--adapter-ip", self.adapter_ip) - if self.adapter_mac: - command += ("--adapter-mac", self.adapter_mac) - if self.router_mac: - command += ("--router-mac", self.router_mac) - if ping: - command += ("--ping",) - else: - if self.ports: - command += ("-p", self.ports) - else: - command += ("--top-ports", str(self.top_ports)) - return command - - def make_targets(self, events): + async def make_targets(self, events, scanned_cache): # convert events into a list of targets, skipping ones that have already been scanned + correlator = RadixTarget() targets = set() - for e in events: + for event in events: # skip events without host - if not e.host: - continue - # skip events that we already scanned - if self.scanned_tracker.search(e.host): - self.debug(f"Skipping {e.host} because it was already scanned") + if not event.host: continue + ips = set() try: # first assume it's an ip address / ip range - host = ipaddress.ip_network(e.host, strict=False) - targets.add(host) - self.scanned_tracker.insert(host, e) + # False == it's not a hostname + ips.add(ipaddress.ip_network(event.host, strict=False)) except Exception: # if it's a hostname, get its IPs from resolved_hosts - hosts = set() - for h in e.resolved_hosts: + for h in event.resolved_hosts: try: - h = ipaddress.ip_network(h, strict=False) - hosts.add(h) + ips.add(ipaddress.ip_network(h, strict=False)) except Exception: continue - for h in hosts: - targets.add(h) - self.scanned_tracker.insert(h, e) - # remove IPv6 addresses if we're not scanning IPv6 - if not self.ipv6_support: - targets = [t for t in targets if t.version != 6] - return targets + for ip in ips: + # remove IPv6 addresses if we're not scanning IPv6 + if not self.ipv6_support and ip.version == 6: + self.debug(f"Not scanning IPv6 address {ip} because we aren't set up for IPv6") + continue + # if we already scanned this one, emit its open port + ip_hash = hash(ip.network_address) + already_found_ports = scanned_cache.get(ip_hash, None) + if already_found_ports is not None: + for port in already_found_ports: + await self.emit_open_port(ip.network_address, port, event) + continue + if not self.scanned_targets.search(ip): + self.scanned_targets.insert(ip, True) + targets.add(ip) + correlator.insert(ip, event) + + return targets, correlator + + async def emit_open_port(self, ip, port, parent_event): + parent_is_dns_name = parent_event.type == "HOST" + if parent_is_dns_name: + host = parent_event.host + else: + host = ip + + if port == 0: + event_data = host + event_type = "DNS_NAME" if parent_is_dns_name else "IP_ADDRESS" + scan_type = "ping" + else: + event_data = self.helpers.make_netloc(host, port) + event_type = "OPEN_TCP_PORT" + scan_type = "TCP SYN" + + await self.emit_event( + event_data, + event_type, + parent=parent_event, + context=f"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}", + ) def parse_json_line(self, line): try: @@ -228,12 +205,21 @@ def parse_json_line(self, line): ip = j.get("ip", "") if not ip: return + ip = self.helpers.make_ip_type(ip) + ip_hash = hash(ip) ports = j.get("ports", []) if not ports: return for p in ports: proto = p.get("proto", "") port_number = p.get("port", 0) + if port_number == 0: + self.alive_hosts_cache[ip_hash] = set(ports) + else: + try: + self.open_ports_cache[ip_hash].add(port_number) + except KeyError: + self.open_ports_cache[ip_hash] = {port_number} if proto == "" or port_number == "": continue yield ip, port_number @@ -251,6 +237,52 @@ def prep_blacklist(self): exclude = ["255.255.255.255/32"] self.exclude_file = self.helpers.tempfile(exclude, pipe=False) + def _build_masscan_command(self, target_file=None, ping=False, dry_run=False, wait=None): + if wait is None: + wait = self.wait + command = ( + "masscan", + "--excludefile", + str(self.exclude_file), + "--rate", + self.rate, + "--wait", + wait, + "--open-only", + "-oJ", + "-", + ) + if target_file is not None: + command += ("-iL", str(target_file)) + if dry_run: + command += ("-p1", "--wait", "0") + else: + if self.adapter: + command += ("--adapter", self.adapter) + if self.adapter_ip: + command += ("--adapter-ip", self.adapter_ip) + if self.adapter_mac: + command += ("--adapter-mac", self.adapter_mac) + if self.router_mac: + command += ("--router-mac", self.router_mac) + if ping: + command += ("--ping",) + else: + if self.ports: + command += ("-p", self.ports) + else: + command += ("--top-ports", str(self.top_ports)) + return command + + def log_masscan_status(self, s): + if "FAIL" in s: + self.warning(s) + self.warning( + f'Masscan failed to detect interface. Recommend passing "adapter_ip", "adapter_mac", and "router_mac" config options to portscan module.' + ) + else: + self.verbose(s) + async def cleanup(self): with suppress(Exception): self.exclude_file.unlink() From 381183914f56b58f0ca0a3fddfee6d1e85609d04 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 28 Jun 2024 18:25:37 -0400 Subject: [PATCH 024/238] allow multiple events to be correlated per IP --- bbot/modules/portscan.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index 7a5f925da..9488641ff 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -119,18 +119,19 @@ async def masscan(self, targets, correlator, ping=False): with open(stats_file, "w") as stats_fh: async for line in self.run_process_live(command, sudo=True, stderr=stats_fh): for ip, port in self.parse_json_line(line): - parent_event = correlator.search(ip) + parent_events = correlator.search(ip) # masscan gets the occasional junk result. this is harmless and # seems to be a side effect of it having its own TCP stack # see https://github.com/robertdavidgraham/masscan/issues/397 - if parent_event is None: + if parent_events is None: self.debug(f"Failed to correlate {ip} to targets") continue - if parent_event.type == "DNS_NAME": - host = parent_event.host - else: - host = ip - yield host, port, parent_event + for parent_event in parent_events: + if parent_event.type == "DNS_NAME": + host = parent_event.host + else: + host = ip + yield host, port, parent_event finally: for file in (stats_file, target_file): file.unlink() @@ -168,9 +169,13 @@ async def make_targets(self, events, scanned_cache): await self.emit_open_port(ip.network_address, port, event) continue if not self.scanned_targets.search(ip): + events_set = correlator.search(ip) + if events_set is None: + events_set = set() + correlator.insert(ip, events_set) self.scanned_targets.insert(ip, True) targets.add(ip) - correlator.insert(ip, event) + events_set.add(event) return targets, correlator From 8b3b3c6f0f1db2c8d7e2361974b81944e4186a49 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 5 Jul 2024 14:23:49 -0400 Subject: [PATCH 025/238] merge from 2.0 --- bbot/core/event/base.py | 18 +++ bbot/core/helpers/async_helpers.py | 2 +- bbot/modules/internal/dns.py | 2 +- bbot/modules/portscan.py | 59 +++++++--- bbot/scanner/target.py | 33 ++---- bbot/test/test_step_1/test_command.py | 2 +- bbot/test/test_step_1/test_target.py | 23 ++-- .../module_tests/test_module_portscan.py | 103 ++++++++++++++++-- 8 files changed, 182 insertions(+), 60 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 30dd0f8a4..dde5ccb3b 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -794,6 +794,24 @@ def type(self, val): self._hash = None self._id = None + @property + def _host_size(self): + """ + Used for sorting events by their host size, so that parent ones (e.g. IP subnets) come first + """ + if self.host: + if isinstance(self.host, str): + # smaller domains should come first + return len(self.host) + else: + try: + # bigger IP subnets should come first + return -self.host.num_addresses + except AttributeError: + # IP addresses default to 1 + return 1 + return 0 + def __iter__(self): """ For dict(event) diff --git a/bbot/core/helpers/async_helpers.py b/bbot/core/helpers/async_helpers.py index dcc510ee4..2152f9bc5 100644 --- a/bbot/core/helpers/async_helpers.py +++ b/bbot/core/helpers/async_helpers.py @@ -122,6 +122,6 @@ def generator(): # Start the event loop in a separate thread thread = BBOTThread(target=lambda: asyncio.run(runner()), daemon=True, custom_name="bbot async_to_sync_gen()") thread.start() - + # Return the generator return generator() diff --git a/bbot/modules/internal/dns.py b/bbot/modules/internal/dns.py index 258756a6f..8671c6fe9 100644 --- a/bbot/modules/internal/dns.py +++ b/bbot/modules/internal/dns.py @@ -106,7 +106,7 @@ async def handle_event(self, event, kwargs): with suppress(ValidationError): if self.scan.whitelisted(host): event_whitelisted = True - # CNAME to a blacklisted resources, means you're blacklisted + # CNAME to a blacklisted resource, means you're blacklisted with suppress(ValidationError): if self.scan.blacklisted(host): dns_tags.add("blacklisted") diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index 9488641ff..0a5dbad84 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -61,13 +61,15 @@ async def setup(self): self.helpers.parse_port_string(self.ports) except ValueError as e: return False, f"Error parsing ports: {e}" + # whether we've finished scanning our original scan targets + self.scanned_initial_targets = False # keeps track of individual scanned IPs and their open ports # this is necessary because we may encounter more hosts with the same IP # and we want to avoid scanning them again self.open_ports_cache = {} self.alive_hosts_cache = {} # keeps track of which IPs/subnets have already been scanned - self.scanned_targets = RadixTarget() + self.scanned_targets = self.helpers.make_target(acl_mode=True) self.prep_blacklist() self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges") # check if we're set up for IPv6 @@ -87,10 +89,18 @@ async def setup(self): return True async def handle_batch(self, *events): + # on our first run, we automatically include all our intial scan targets + if not self.scanned_initial_targets: + self.scanned_initial_targets = True + events = set(events) + events.update( + set([e for e in self.scan.target.seeds.events if e.type in ("DNS_NAME", "IP_ADDRESS", "IP_RANGE")]) + ) + # ping scan if self.ping_scan: ping_targets, ping_correlator = await self.make_targets(events, self.alive_hosts_cache) - syn_targets, syn_correlator = RadixTarget(self.open_ports_cache) + syn_targets, syn_correlator = self.make_targets([], self.open_ports_cache) async for alive_host, _, parent_event in self.masscan(ping_targets, ping_correlator, ping=True): # port 0 means icmp ping response await self.emit_open_port(alive_host, 0, parent_event) @@ -137,10 +147,12 @@ async def masscan(self, targets, correlator, ping=False): file.unlink() async def make_targets(self, events, scanned_cache): - # convert events into a list of targets, skipping ones that have already been scanned + """ + Convert events into a list of targets, skipping ones that have already been scanned + """ correlator = RadixTarget() targets = set() - for event in events: + for event in sorted(events, key=lambda e: e._host_size): # skip events without host if not event.host: continue @@ -156,27 +168,38 @@ async def make_targets(self, events, scanned_cache): ips.add(ipaddress.ip_network(h, strict=False)) except Exception: continue + for ip in ips: # remove IPv6 addresses if we're not scanning IPv6 if not self.ipv6_support and ip.version == 6: self.debug(f"Not scanning IPv6 address {ip} because we aren't set up for IPv6") continue - # if we already scanned this one, emit its open port - ip_hash = hash(ip.network_address) - already_found_ports = scanned_cache.get(ip_hash, None) - if already_found_ports is not None: - for port in already_found_ports: - await self.emit_open_port(ip.network_address, port, event) - continue - if not self.scanned_targets.search(ip): - events_set = correlator.search(ip) - if events_set is None: - events_set = set() - correlator.insert(ip, events_set) - self.scanned_targets.insert(ip, True) - targets.add(ip) + + # check if we already found open ports on this IP + if event.type != "IP_RANGE": + ip_hash = hash(ip.network_address) + already_found_ports = scanned_cache.get(ip_hash, None) + if already_found_ports is not None: + # if so, no need to scan this one. emit them and move on + for port in already_found_ports: + await self.emit_open_port(event.host, port, event) + continue + + # build a correlation from the IP back to its original parent event + events_set = correlator.search(ip) + if events_set is None: + correlator.insert(ip, {event}) + else: events_set.add(event) + # has this IP already been scanned? + if not self.scanned_targets.get(ip): + # if not, add it to targets! + self.scanned_targets.add(ip) + targets.add(ip) + else: + self.debug(f"Skipping {ip} because it's already been scanned") + return targets, correlator async def emit_open_port(self, ip, port, parent_event): diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 23e68e5f4..8b88882ce 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -376,12 +376,13 @@ def copy(self): self_copy._radix = copy.copy(self._radix) return self_copy - def get(self, host): + def get(self, host, single=True): """ Gets the event associated with the specified host from the target's radix tree. Args: host (Event, Target, or str): The hostname, IP, URL, or event to look for. + single (bool): Whether to return a single event. If False, return all events matching the host Returns: Event or None: Returns the Event object associated with the given host if it exists, otherwise returns None. @@ -397,15 +398,14 @@ def get(self, host): - The method returns the first event that matches the given host. - If `strict_scope` is False, it will also consider parent domains and IP ranges. """ - try: event = make_event(host, dummy=True) except ValidationError: return if event.host: - return self.get_host(event.host) + return self.get_host(event.host, single=single) - def get_host(self, host): + def get_host(self, host, single=True): """ A more efficient version of .get() that only accepts hostnames and IP addresses """ @@ -413,31 +413,22 @@ def get_host(self, host): with suppress(KeyError, StopIteration): result = self._radix.search(host) if result is not None: + ret = set() for event in result: # if the result is a dns name and strict scope is enabled if isinstance(event.host, str) and self.strict_scope: # if the result doesn't exactly equal the host, abort if event.host != host: return - return event - - def _len_event(self, event): - """ - Used for sorting events by their length, so that bigger ones (e.g. IP subnets) are added first - """ - try: - # smaller domains should come first - return len(event.host) - except TypeError: - try: - # bigger IP subnets should come first - return -event.host.num_addresses - except AttributeError: - # IP addresses default to 1 - return 1 + if single: + return event + else: + ret.add(event) + if ret and not single: + return ret def _sort_events(self, events): - return sorted(events, key=self._len_event) + return sorted(events, key=lambda x: x._host_size) def _make_events(self, targets): events = [] diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py index b0afdcc78..2a8d21e6b 100644 --- a/bbot/test/test_step_1/test_command.py +++ b/bbot/test/test_step_1/test_command.py @@ -116,7 +116,7 @@ async def test_command(bbot_scanner): assert not lines # test sudo + existence of environment variables - scan1.load_modules() + await scan1.load_modules() path_parts = os.environ.get("PATH", "").split(":") assert "/tmp/.bbot_test/tools" in path_parts run_lines = (await scan1.helpers.run(["env"])).stdout.splitlines() diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 402ac77c2..23175607f 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -239,13 +239,13 @@ async def test_target(bbot_scanner): grandparent_domain = scan.make_event("www.evilcorp.com", dummy=True) greatgrandparent_domain = scan.make_event("api.www.evilcorp.com", dummy=True) target = Target() - assert target._len_event(big_subnet) == -256 - assert target._len_event(medium_subnet) == -16 - assert target._len_event(small_subnet) == -4 - assert target._len_event(ip_event) == 1 - assert target._len_event(parent_domain) == 12 - assert target._len_event(grandparent_domain) == 16 - assert target._len_event(greatgrandparent_domain) == 20 + assert big_subnet._host_size == -256 + assert medium_subnet._host_size == -16 + assert small_subnet._host_size == -4 + assert ip_event._host_size == 1 + assert parent_domain._host_size == 12 + assert grandparent_domain._host_size == 16 + assert greatgrandparent_domain._host_size == 20 events = [ big_subnet, medium_subnet, @@ -289,3 +289,12 @@ async def test_target(bbot_scanner): assert "www.evilcorp.co.uk" in target assert not "api.evilcorp.co.uk" in target assert not "api.www.evilcorp.co.uk" in target + + # test 'single' boolean argument + target = Target("http://evilcorp.com", "evilcorp.com:443") + assert "www.evilcorp.com" in target + event = target.get("www.evilcorp.com") + assert event.host == "evilcorp.com" + events = target.get("www.evilcorp.com", single=False) + assert len(events) == 2 + assert set([e.data for e in events]) == {"http://evilcorp.com/", "evilcorp.com:443"} diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py index 93c55f715..0abbd13a2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_portscan.py +++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py @@ -2,24 +2,63 @@ class TestPortscan(ModuleTestBase): - targets = ["8.8.8.8/32"] + targets = [ + "www.evilcorp.com", + "evilcorp.com", + "8.8.8.8/32", + "8.8.8.8/24", + "8.8.4.4", + "asdf.evilcorp.net", + "8.8.4.4/24", + ] scan_name = "test_portscan" - config_overrides = {"modules": {"portscan": {"ports": "443", "wait": 1}}} + config_overrides = {"modules": {"portscan": {"ports": "443", "wait": 1}}, "dns_resolution": True} - masscan_output = """{ "ip": "8.8.8.8", "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }""" + masscan_output_1 = """{ "ip": "8.8.8.8", "timestamp": "1680197558", "ports": [ {"port": 443, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }""" + masscan_output_2 = """{ "ip": "8.8.4.5", "timestamp": "1680197558", "ports": [ {"port": 80, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }""" + masscan_output_3 = """{ "ip": "8.8.4.6", "timestamp": "1680197558", "ports": [ {"port": 631, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }""" async def setup_after_prep(self, module_test): - self.masscan_run = False + + from bbot.modules.base import BaseModule + + class DummyModule(BaseModule): + _name = "dummy_module" + watched_events = ["*"] + + async def handle_event(self, event): + if event.type == "DNS_NAME": + if "dummy" not in event.host: + await self.emit_event(f"dummy.{event.data}", "DNS_NAME", parent=event) + + module_test.scan.modules["dummy_module"] = DummyModule(module_test.scan) + + await module_test.mock_dns( + { + "www.evilcorp.com": {"A": ["8.8.8.8"]}, + "evilcorp.com": {"A": ["8.8.8.8"]}, + "asdf.evilcorp.net": {"A": ["8.8.4.5"]}, + "dummy.asdf.evilcorp.net": {"A": ["8.8.4.5"]}, + "dummy.evilcorp.com": {"A": ["8.8.4.6"]}, + "dummy.www.evilcorp.com": {"A": ["8.8.4.4"]}, + } + ) + + self.targets_scanned = [] + self.masscan_runs = 0 async def run_masscan(command, *args, **kwargs): if "masscan" in command[:2]: targets = open(command[11]).read().splitlines() + self.targets_scanned += targets + self.masscan_runs += 1 yield "[" - for l in self.masscan_output.splitlines(): - if "8.8.8.8/32" in targets: - yield self.masscan_output + if "8.8.8.0/24" in targets: + yield self.masscan_output_1 + if "8.8.4.0/24" in targets: + yield self.masscan_output_2 + yield self.masscan_output_3 yield "]" - self.masscan_run = True else: async for l in module_test.scan.helpers.run_live(command, *args, **kwargs): yield l @@ -27,6 +66,48 @@ async def run_masscan(command, *args, **kwargs): module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", run_masscan) def check(self, module_test, events): - assert self.masscan_run == True - assert any(e.type == "IP_ADDRESS" and e.data == "8.8.8.8" for e in events), "No IP_ADDRESS emitted" - assert any(e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443" for e in events), "No OPEN_TCP_PORT emitted" + assert set(self.targets_scanned) == {"8.8.8.0/24", "8.8.4.0/24"} + assert self.masscan_runs == 1 + assert 1 == len( + [e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com" and str(e.module) == "TARGET"] + ) + assert 1 == len( + [e for e in events if e.type == "DNS_NAME" and e.data == "www.evilcorp.com" and str(e.module) == "TARGET"] + ) + assert 1 == len( + [e for e in events if e.type == "DNS_NAME" and e.data == "asdf.evilcorp.net" and str(e.module) == "TARGET"] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "dummy.evilcorp.com" and str(e.module) == "dummy_module" + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "dummy.www.evilcorp.com" and str(e.module) == "dummy_module" + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "dummy.asdf.evilcorp.net" and str(e.module) == "dummy_module" + ] + ) + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 3 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 3 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 3 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 3 + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.5:80"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.6:631"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "evilcorp.com:443"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "www.evilcorp.com:443"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "asdf.evilcorp.net:80"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.asdf.evilcorp.net:80"]) + assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.evilcorp.com:631"]) + assert not any([e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "dummy.www.evilcorp.com"]) From af03208d16145a129eb019995055ddd7651ec31c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 16:36:45 -0400 Subject: [PATCH 026/238] better tests for portscan module --- bbot/modules/portscan.py | 50 ++++++------- .../module_tests/test_module_portscan.py | 71 ++++++++++++++++--- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index 0a5dbad84..de4d4a41e 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -66,10 +66,10 @@ async def setup(self): # keeps track of individual scanned IPs and their open ports # this is necessary because we may encounter more hosts with the same IP # and we want to avoid scanning them again - self.open_ports_cache = {} - self.alive_hosts_cache = {} + self.open_port_cache = {} # keeps track of which IPs/subnets have already been scanned - self.scanned_targets = self.helpers.make_target(acl_mode=True) + self.syn_scanned = self.helpers.make_target(acl_mode=True) + self.ping_scanned = self.helpers.make_target(acl_mode=True) self.prep_blacklist() self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges") # check if we're set up for IPv6 @@ -99,14 +99,15 @@ async def handle_batch(self, *events): # ping scan if self.ping_scan: - ping_targets, ping_correlator = await self.make_targets(events, self.alive_hosts_cache) - syn_targets, syn_correlator = self.make_targets([], self.open_ports_cache) + ping_targets, ping_correlator = await self.make_targets(events, self.ping_scanned) + ping_events = [] async for alive_host, _, parent_event in self.masscan(ping_targets, ping_correlator, ping=True): # port 0 means icmp ping response - await self.emit_open_port(alive_host, 0, parent_event) - syn_targets.insert(ipaddress.ip_network(alive_host, strict=False), parent_event) + ping_event = await self.emit_open_port(alive_host, 0, parent_event) + ping_events.append(ping_event) + syn_targets, syn_correlator = await self.make_targets(ping_events, self.syn_scanned) else: - syn_targets, syn_correlator = await self.make_targets(events, self.open_ports_cache) + syn_targets, syn_correlator = await self.make_targets(events, self.syn_scanned) # TCP SYN scan if not self.ping_only: @@ -136,17 +137,20 @@ async def masscan(self, targets, correlator, ping=False): if parent_events is None: self.debug(f"Failed to correlate {ip} to targets") continue + emitted_hosts = set() for parent_event in parent_events: if parent_event.type == "DNS_NAME": host = parent_event.host else: host = ip - yield host, port, parent_event + if host not in emitted_hosts: + yield host, port, parent_event + emitted_hosts.add(host) finally: for file in (stats_file, target_file): file.unlink() - async def make_targets(self, events, scanned_cache): + async def make_targets(self, events, scanned_tracker): """ Convert events into a list of targets, skipping ones that have already been scanned """ @@ -178,12 +182,11 @@ async def make_targets(self, events, scanned_cache): # check if we already found open ports on this IP if event.type != "IP_RANGE": ip_hash = hash(ip.network_address) - already_found_ports = scanned_cache.get(ip_hash, None) + already_found_ports = self.open_port_cache.get(ip_hash, None) if already_found_ports is not None: - # if so, no need to scan this one. emit them and move on + # if so, emit them for port in already_found_ports: await self.emit_open_port(event.host, port, event) - continue # build a correlation from the IP back to its original parent event events_set = correlator.search(ip) @@ -193,9 +196,9 @@ async def make_targets(self, events, scanned_cache): events_set.add(event) # has this IP already been scanned? - if not self.scanned_targets.get(ip): + if not scanned_tracker.get(ip): # if not, add it to targets! - self.scanned_targets.add(ip) + scanned_tracker.add(ip) targets.add(ip) else: self.debug(f"Skipping {ip} because it's already been scanned") @@ -203,7 +206,7 @@ async def make_targets(self, events, scanned_cache): return targets, correlator async def emit_open_port(self, ip, port, parent_event): - parent_is_dns_name = parent_event.type == "HOST" + parent_is_dns_name = parent_event.type == "DNS_NAME" if parent_is_dns_name: host = parent_event.host else: @@ -218,12 +221,14 @@ async def emit_open_port(self, ip, port, parent_event): event_type = "OPEN_TCP_PORT" scan_type = "TCP SYN" - await self.emit_event( + event = self.make_event( event_data, event_type, parent=parent_event, context=f"{{module}} executed a {scan_type} scan against {parent_event.data} and found: {{event.type}}: {{event.data}}", ) + await self.emit_event(event) + return event def parse_json_line(self, line): try: @@ -241,13 +246,10 @@ def parse_json_line(self, line): for p in ports: proto = p.get("proto", "") port_number = p.get("port", 0) - if port_number == 0: - self.alive_hosts_cache[ip_hash] = set(ports) - else: - try: - self.open_ports_cache[ip_hash].add(port_number) - except KeyError: - self.open_ports_cache[ip_hash] = {port_number} + try: + self.open_port_cache[ip_hash].add(port_number) + except KeyError: + self.open_port_cache[ip_hash] = {port_number} if proto == "" or port_number == "": continue yield ip, port_number diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py index 0abbd13a2..4d2a66ee4 100644 --- a/bbot/test/test_step_2/module_tests/test_module_portscan.py +++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py @@ -18,6 +18,8 @@ class TestPortscan(ModuleTestBase): masscan_output_2 = """{ "ip": "8.8.4.5", "timestamp": "1680197558", "ports": [ {"port": 80, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }""" masscan_output_3 = """{ "ip": "8.8.4.6", "timestamp": "1680197558", "ports": [ {"port": 631, "proto": "tcp", "status": "open", "reason": "syn-ack", "ttl": 54} ] }""" + masscan_output_ping = """{ "ip": "8.8.8.8", "timestamp": "1719862594", "ports": [ {"port": 0, "proto": "icmp", "status": "open", "reason": "none", "ttl": 54} ] }""" + async def setup_after_prep(self, module_test): from bbot.modules.base import BaseModule @@ -44,20 +46,27 @@ async def handle_event(self, event): } ) - self.targets_scanned = [] - self.masscan_runs = 0 + self.syn_scanned = [] + self.ping_scanned = [] + self.syn_runs = 0 + self.ping_runs = 0 async def run_masscan(command, *args, **kwargs): if "masscan" in command[:2]: targets = open(command[11]).read().splitlines() - self.targets_scanned += targets - self.masscan_runs += 1 yield "[" - if "8.8.8.0/24" in targets: - yield self.masscan_output_1 - if "8.8.4.0/24" in targets: - yield self.masscan_output_2 - yield self.masscan_output_3 + if "--ping" in command: + self.ping_runs += 1 + self.ping_scanned += targets + yield self.masscan_output_ping + else: + self.syn_runs += 1 + self.syn_scanned += targets + if "8.8.8.0/24" in targets or "8.8.8.8/32" in targets: + yield self.masscan_output_1 + if "8.8.4.0/24" in targets: + yield self.masscan_output_2 + yield self.masscan_output_3 yield "]" else: async for l in module_test.scan.helpers.run_live(command, *args, **kwargs): @@ -66,8 +75,10 @@ async def run_masscan(command, *args, **kwargs): module_test.monkeypatch.setattr(module_test.scan.helpers, "run_live", run_masscan) def check(self, module_test, events): - assert set(self.targets_scanned) == {"8.8.8.0/24", "8.8.4.0/24"} - assert self.masscan_runs == 1 + assert set(self.syn_scanned) == {"8.8.8.0/24", "8.8.4.0/24"} + assert set(self.ping_scanned) == set() + assert self.syn_runs == 1 + assert self.ping_runs == 0 assert 1 == len( [e for e in events if e.type == "DNS_NAME" and e.data == "evilcorp.com" and str(e.module) == "TARGET"] ) @@ -111,3 +122,41 @@ def check(self, module_test, events): assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.asdf.evilcorp.net:80"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "dummy.evilcorp.com:631"]) assert not any([e for e in events if e.type == "OPEN_TCP_PORT" and e.host == "dummy.www.evilcorp.com"]) + + +class TestPortscanPingFirst(TestPortscan): + modules_overrides = {"portscan"} + config_overrides = { + "modules": {"portscan": {"ports": "443", "wait": 1, "ping_first": True}}, + "dns_resolution": True, + } + + def check(self, module_test, events): + assert set(self.syn_scanned) == {"8.8.8.8/32"} + assert set(self.ping_scanned) == {"8.8.8.0/24", "8.8.4.0/24"} + assert self.syn_runs == 1 + assert self.ping_runs == 1 + open_port_events = [e for e in events if e.type == "OPEN_TCP_PORT"] + assert len(open_port_events) == 3 + assert set([e.data for e in open_port_events]) == {"8.8.8.8:443", "evilcorp.com:443", "www.evilcorp.com:443"} + + +class TestPortscanPingOnly(TestPortscan): + modules_overrides = {"portscan"} + config_overrides = { + "modules": {"portscan": {"ports": "443", "wait": 1, "ping_only": True}}, + "dns_resolution": True, + } + + targets = ["8.8.8.8/24", "8.8.4.4/24"] + + def check(self, module_test, events): + assert set(self.syn_scanned) == set() + assert set(self.ping_scanned) == {"8.8.8.0/24", "8.8.4.0/24"} + assert self.syn_runs == 0 + assert self.ping_runs == 1 + open_port_events = [e for e in events if e.type == "OPEN_TCP_PORT"] + assert len(open_port_events) == 0 + ip_events = [e for e in events if e.type == "IP_ADDRESS"] + assert len(ip_events) == 1 + assert set([e.data for e in ip_events]) == {"8.8.8.8"} From b40977a86b9fb4c8956a349f40875fc2c361dd46 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 17:21:24 -0400 Subject: [PATCH 027/238] fixed wafw00f bug, wrote test for it --- bbot/modules/wafw00f.py | 3 ++- .../module_tests/test_module_wafw00f.py | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index ac408765c..a9c7aca76 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -55,11 +55,12 @@ async def handle_event(self, event): if self.config.get("generic_detect") == True: generic = await self.helpers.run_in_executor(WW.genericdetect) if generic: + waf = "generic detection" await self.emit_event( { "host": str(event.host), "url": url, - "waf": "generic detection", + "waf": waf, "info": WW.knowledge["generic"]["reason"], }, "WAF", diff --git a/bbot/test/test_step_2/module_tests/test_module_wafw00f.py b/bbot/test/test_step_2/module_tests/test_module_wafw00f.py index 39cb43c13..892d892ff 100644 --- a/bbot/test/test_step_2/module_tests/test_module_wafw00f.py +++ b/bbot/test/test_step_2/module_tests/test_module_wafw00f.py @@ -1,5 +1,7 @@ from .base import ModuleTestBase +from werkzeug.wrappers import Response + class TestWafw00f(ModuleTestBase): targets = ["http://127.0.0.1:8888"] @@ -28,3 +30,21 @@ async def setup_after_prep(self, module_test): def check(self, module_test, events): assert not any(e.type == "WAF" for e in events) + + +class TestWafw00f_genericdetection(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "wafw00f"] + + async def setup_after_prep(self, module_test): + def handler(request): + if "SLEEP" in request.url: + return Response("nope", status=403) + return Response("yep") + + module_test.httpserver.expect_request("/").respond_with_handler(handler) + + def check(self, module_test, events): + waf_events = [e for e in events if e.type == "WAF"] + assert len(waf_events) == 1 + assert waf_events[0].data["waf"] == "generic detection" From 174e241aa29797aae52620fa8a5830d5d2abbe1e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 22:48:10 -0400 Subject: [PATCH 028/238] set thread title --- bbot/core/core.py | 3 --- bbot/core/helpers/process.py | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bbot/core/core.py b/bbot/core/core.py index 05a8af297..e7eacf18d 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -187,10 +187,7 @@ def files_config(self): def create_process(self, *args, **kwargs): if os.environ.get("BBOT_TESTING", "") == "True": - # if threading.current_thread() is threading.main_thread(): process = self.create_thread(*args, **kwargs) - # else: - # raise BBOTError(f"Tried to start server from process {self.process_name}") else: if self.process_name == "MainProcess": from .helpers.process import BBOTProcess diff --git a/bbot/core/helpers/process.py b/bbot/core/helpers/process.py index a00a93994..7f3a23849 100644 --- a/bbot/core/helpers/process.py +++ b/bbot/core/helpers/process.py @@ -19,9 +19,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def run(self): - from setproctitle import setproctitle + from setproctitle import setthreadtitle - setproctitle(str(self.custom_name)) + setthreadtitle(str(self.custom_name)) super().run() From 48835b0d2877f3368dbbeb84009991df6d4001af Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 22:51:26 -0400 Subject: [PATCH 029/238] close context --- bbot/core/engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index 1674a7e6b..cff06f508 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -191,6 +191,7 @@ async def shutdown(self): self.cleanup() def cleanup(self): + self.context.close() # delete socket file on exit self.socket_path.unlink(missing_ok=True) @@ -336,3 +337,5 @@ async def worker(self): finally: with suppress(Exception): self.socket.close() + with suppress(Exception): + self.context.close() From 9547832d0da39ec014d4842b3df443f0c9754807 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 1 Jul 2024 22:53:59 -0400 Subject: [PATCH 030/238] destroyyyyy --- bbot/core/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/core/engine.py b/bbot/core/engine.py index cff06f508..7d735133f 100644 --- a/bbot/core/engine.py +++ b/bbot/core/engine.py @@ -191,7 +191,7 @@ async def shutdown(self): self.cleanup() def cleanup(self): - self.context.close() + self.context.destroy() # delete socket file on exit self.socket_path.unlink(missing_ok=True) @@ -338,4 +338,4 @@ async def worker(self): with suppress(Exception): self.socket.close() with suppress(Exception): - self.context.close() + self.context.destroy() From 1487f256f7e40ebbac8ee989496348f8e58eb8e6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Jul 2024 13:45:45 -0400 Subject: [PATCH 031/238] add libpcap to masscan deps --- bbot/core/shared_deps.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py index 3f67c354d..377ec3af3 100644 --- a/bbot/core/shared_deps.py +++ b/bbot/core/shared_deps.py @@ -113,9 +113,17 @@ DEP_MASSCAN = [ { - "name": "install dev tools", - "package": {"name": ["gcc", "git", "make"], "state": "present"}, + "name": "install os deps (Debian)", + "package": {"name": ["gcc", "git", "make", "libpcap-0.8"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian'", + "ignore_errors": True, + }, + { + "name": "install dev tools (Non-Debian)", + "package": {"name": ["gcc", "git", "make", "libpcap"], "state": "present"}, "become": True, + "when": "ansible_facts['os_family'] != 'Debian'", "ignore_errors": True, }, { From c0e307474ac2aa4578bd31a53b210d25b263b2f1 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Jul 2024 13:46:37 -0400 Subject: [PATCH 032/238] fix package name --- bbot/core/shared_deps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py index 377ec3af3..995505d11 100644 --- a/bbot/core/shared_deps.py +++ b/bbot/core/shared_deps.py @@ -114,7 +114,7 @@ DEP_MASSCAN = [ { "name": "install os deps (Debian)", - "package": {"name": ["gcc", "git", "make", "libpcap-0.8"], "state": "present"}, + "package": {"name": ["gcc", "git", "make", "libpcap0.8"], "state": "present"}, "become": True, "when": "ansible_facts['os_family'] == 'Debian'", "ignore_errors": True, From 2b319b278598ceda80b317134259fa54d0d5aaa7 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Jul 2024 13:52:28 -0400 Subject: [PATCH 033/238] switch to -dev --- bbot/core/shared_deps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/shared_deps.py b/bbot/core/shared_deps.py index 995505d11..b8c58e0b1 100644 --- a/bbot/core/shared_deps.py +++ b/bbot/core/shared_deps.py @@ -114,7 +114,7 @@ DEP_MASSCAN = [ { "name": "install os deps (Debian)", - "package": {"name": ["gcc", "git", "make", "libpcap0.8"], "state": "present"}, + "package": {"name": ["gcc", "git", "make", "libpcap0.8-dev"], "state": "present"}, "become": True, "when": "ansible_facts['os_family'] == 'Debian'", "ignore_errors": True, From 022ec539f3cce3c47e1e7f1ed7f147342bc27167 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Jul 2024 14:48:44 -0400 Subject: [PATCH 034/238] fix masscan top ports issue --- bbot/core/helpers/misc.py | 25 ++++++++++++++++++++++++- bbot/modules/portscan.py | 2 +- bbot/test/test_step_1/test_helpers.py | 12 ++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 446166a0e..a9bfc1ef3 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2772,4 +2772,27 @@ def calculate_entropy(data): frequency[byte] = 1 data_len = len(data) entropy = -sum((count / data_len) * math.log2(count / data_len) for count in frequency.values()) - return entropy \ No newline at end of file + return entropy +top_ports_cache = None + + +def top_tcp_ports(n, as_string=False): + """ + Returns the top *n* TCP ports as evaluated by nmap + """ + top_ports_file = Path(__file__).parent.parent.parent / "wordlists" / "top_open_ports_nmap.txt" + + global top_ports_cache + if top_ports_cache is None: + # Read the open ports from the file + with open(top_ports_file, "r") as f: + top_ports_cache = [int(line.strip()) for line in f] + + # If n is greater than the length of the ports list, add remaining ports from range(1, 65536) + unique_ports = set(top_ports_cache) + top_ports_cache.extend([port for port in range(1, 65536) if port not in unique_ports]) + + top_ports = top_ports_cache[:n] + if as_string: + return ",".join([str(s) for s in top_ports]) + return top_ports diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index de4d4a41e..333cd0d6a 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -301,7 +301,7 @@ def _build_masscan_command(self, target_file=None, ping=False, dry_run=False, wa if self.ports: command += ("-p", self.ports) else: - command += ("--top-ports", str(self.top_ports)) + command += ("-p", self.helpers.top_tcp_ports(self.top_ports, as_string=True)) return command def log_masscan_status(self, s): diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 1cfea1f0d..a08a4c270 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -387,6 +387,18 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): assert helpers.validators.soft_validate("!@#$", "port") == False with pytest.raises(ValueError): helpers.validators.validate_port("asdf") + # top tcp ports + top_tcp_ports = helpers.top_tcp_ports(100) + assert len(top_tcp_ports) == 100 + assert len(set(top_tcp_ports)) == 100 + top_tcp_ports = helpers.top_tcp_ports(800000) + assert top_tcp_ports[:10] == [80, 23, 443, 21, 22, 25, 3389, 110, 445, 139] + assert top_tcp_ports[-10:] == [65526, 65527, 65528, 65529, 65530, 65531, 65532, 65533, 65534, 65535] + assert len(top_tcp_ports) == 65535 + assert len(set(top_tcp_ports)) == 65535 + assert all([isinstance(i, int) for i in top_tcp_ports]) + top_tcp_ports = helpers.top_tcp_ports(10, as_string=True) + assert top_tcp_ports == "80,23,443,21,22,25,3389,110,445,139" # urls assert helpers.validators.validate_url(" httP://evilcorP.com/asdf?a=b&c=d#e") == "http://evilcorp.com/asdf" assert ( From 4c3561d1aeefd76539e5ffa57eca9d53d5171d7e Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Jul 2024 15:34:18 -0400 Subject: [PATCH 035/238] add top ports --- bbot/wordlists/top_open_ports_nmap.txt | 8377 ++++++++++++++++++++++++ 1 file changed, 8377 insertions(+) create mode 100644 bbot/wordlists/top_open_ports_nmap.txt diff --git a/bbot/wordlists/top_open_ports_nmap.txt b/bbot/wordlists/top_open_ports_nmap.txt new file mode 100644 index 000000000..30f1bcefd --- /dev/null +++ b/bbot/wordlists/top_open_ports_nmap.txt @@ -0,0 +1,8377 @@ +80 +23 +443 +21 +22 +25 +3389 +110 +445 +139 +143 +53 +135 +3306 +8080 +1723 +111 +995 +993 +5900 +1025 +587 +8888 +199 +1720 +465 +548 +113 +81 +6001 +10000 +514 +5060 +179 +1026 +2000 +8443 +8000 +32768 +554 +26 +1433 +49152 +2001 +515 +8008 +49154 +1027 +5666 +646 +5000 +5631 +631 +49153 +8081 +2049 +88 +79 +5800 +106 +2121 +1110 +49155 +6000 +513 +990 +5357 +427 +49156 +543 +544 +5101 +144 +7 +389 +8009 +3128 +444 +9999 +5009 +7070 +5190 +3000 +5432 +1900 +3986 +13 +1029 +9 +5051 +6646 +49157 +1028 +873 +1755 +2717 +4899 +9100 +119 +37 +1000 +3001 +5001 +82 +10010 +1030 +9090 +2107 +1024 +2103 +6004 +1801 +5050 +19 +8031 +1041 +255 +1049 +1048 +2967 +1053 +3703 +1056 +1065 +1064 +1054 +17 +808 +3689 +1031 +1044 +1071 +5901 +100 +9102 +8010 +2869 +1039 +5120 +4001 +9000 +2105 +636 +1038 +2601 +1 +7000 +1066 +1069 +625 +311 +280 +254 +4000 +1761 +5003 +2002 +2005 +1998 +1032 +1050 +6112 +3690 +1521 +2161 +6002 +1080 +2401 +4045 +902 +7937 +787 +1058 +2383 +32771 +1033 +1040 +1059 +50000 +5555 +10001 +1494 +593 +2301 +3 +3268 +7938 +1234 +1022 +1074 +8002 +1036 +1035 +9001 +1037 +464 +497 +1935 +6666 +2003 +6543 +1352 +24 +3269 +1111 +407 +500 +20 +2006 +3260 +15000 +1218 +1034 +4444 +264 +2004 +33 +1042 +42510 +999 +3052 +1023 +1068 +222 +7100 +888 +563 +1717 +2008 +992 +32770 +32772 +7001 +8082 +2007 +5550 +2009 +5801 +1043 +512 +2701 +7019 +50001 +1700 +4662 +2065 +2010 +42 +9535 +2602 +3333 +161 +5100 +5002 +2604 +4002 +6059 +1047 +8192 +8193 +2702 +6789 +9595 +1051 +9594 +9593 +16993 +16992 +5226 +5225 +32769 +3283 +1052 +8194 +1055 +1062 +9415 +8701 +8652 +8651 +8089 +65389 +65000 +64680 +64623 +55600 +55555 +52869 +35500 +33354 +23502 +20828 +1311 +1060 +4443 +1067 +13782 +5902 +366 +9050 +1002 +85 +5500 +5431 +1864 +1863 +8085 +51103 +49999 +45100 +10243 +49 +6667 +90 +27000 +1503 +6881 +1500 +8021 +340 +5566 +8088 +2222 +9071 +8899 +6005 +9876 +1501 +5102 +32774 +32773 +9101 +5679 +163 +648 +146 +1666 +901 +83 +9207 +8001 +8083 +8084 +5004 +3476 +5214 +14238 +12345 +912 +30 +2605 +2030 +6 +541 +8007 +3005 +4 +1248 +2500 +880 +306 +4242 +1097 +9009 +2525 +1086 +1088 +8291 +52822 +6101 +900 +7200 +2809 +800 +32775 +12000 +1083 +211 +987 +705 +20005 +711 +13783 +6969 +3071 +5269 +5222 +1085 +1046 +5986 +5985 +5987 +5989 +5988 +2190 +3301 +11967 +8600 +3766 +7627 +8087 +30000 +9010 +7741 +14000 +3367 +1099 +1098 +3031 +2718 +6580 +15002 +4129 +6901 +3827 +3580 +2144 +8181 +3801 +1718 +2811 +9080 +2135 +1045 +2399 +3017 +10002 +1148 +9002 +8873 +2875 +9011 +5718 +8086 +20000 +3998 +2607 +11110 +4126 +9618 +2381 +1096 +3300 +3351 +1073 +8333 +3784 +5633 +15660 +6123 +3211 +1078 +5910 +5911 +3659 +3551 +2260 +2160 +2100 +16001 +3325 +3323 +1104 +9968 +9503 +9502 +9485 +9290 +9220 +8994 +8649 +8222 +7911 +7625 +7106 +65129 +63331 +6156 +6129 +60020 +5962 +5961 +5960 +5959 +5925 +5877 +5825 +5810 +58080 +57294 +50800 +50006 +50003 +49160 +49159 +49158 +48080 +40193 +34573 +34572 +34571 +3404 +33899 +32782 +32781 +31038 +30718 +28201 +27715 +25734 +24800 +22939 +21571 +20221 +20031 +19842 +19801 +19101 +17988 +1783 +16018 +16016 +15003 +14442 +13456 +10629 +10628 +10626 +10621 +10617 +10616 +10566 +10025 +10024 +10012 +1169 +5030 +5414 +1057 +6788 +1947 +1094 +1075 +1108 +4003 +1081 +1093 +4449 +1687 +1840 +1100 +1063 +1061 +9900 +1107 +1106 +9500 +20222 +7778 +1077 +1310 +2119 +2492 +1070 +8400 +1272 +6389 +7777 +1072 +1079 +1082 +8402 +89 +691 +1001 +32776 +1999 +212 +2020 +6003 +7002 +2998 +50002 +3372 +898 +5510 +32 +2033 +99 +749 +425 +5903 +43 +5405 +6106 +13722 +6502 +7007 +458 +9666 +8100 +3737 +5298 +1152 +8090 +2191 +3011 +1580 +9877 +5200 +3851 +3371 +3370 +3369 +7402 +5054 +3918 +3077 +7443 +3493 +3828 +1186 +2179 +1183 +19315 +19283 +3995 +5963 +1124 +8500 +1089 +10004 +2251 +1087 +5280 +3871 +3030 +62078 +5904 +9091 +4111 +1334 +3261 +2522 +5859 +1247 +9944 +9943 +9110 +8654 +8254 +8180 +8011 +7512 +7435 +7103 +61900 +61532 +5922 +5915 +5822 +56738 +55055 +51493 +50636 +50389 +49175 +49165 +49163 +3546 +32784 +27355 +27353 +27352 +24444 +19780 +18988 +16012 +15742 +10778 +4006 +2126 +4446 +3880 +1782 +1296 +9998 +9040 +32779 +1021 +32777 +2021 +32778 +616 +666 +700 +5802 +4321 +545 +1524 +1112 +49400 +84 +38292 +2040 +32780 +3006 +2111 +1084 +1600 +2048 +2638 +9111 +6699 +16080 +6547 +6007 +1533 +5560 +2106 +1443 +667 +720 +2034 +555 +801 +6025 +3221 +3826 +9200 +2608 +4279 +7025 +11111 +3527 +1151 +8200 +8300 +6689 +9878 +10009 +8800 +5730 +2394 +2393 +2725 +6566 +9081 +5678 +5906 +3800 +4550 +5080 +1201 +3168 +3814 +1862 +1114 +6510 +3905 +8383 +3914 +3971 +3809 +5033 +7676 +3517 +4900 +3869 +9418 +2909 +3878 +8042 +1091 +1090 +3920 +6567 +1138 +3945 +1175 +10003 +3390 +5907 +3889 +1131 +8292 +5087 +1119 +1117 +4848 +7800 +16000 +3324 +3322 +5221 +4445 +9917 +9575 +9099 +9003 +8290 +8099 +8093 +8045 +7921 +7920 +7496 +6839 +6792 +6779 +6692 +6565 +60443 +5952 +5950 +5862 +5850 +5815 +5811 +57797 +56737 +5544 +55056 +5440 +54328 +54045 +52848 +52673 +50500 +50300 +49176 +49167 +49161 +44501 +44176 +41511 +40911 +32785 +32783 +30951 +27356 +26214 +25735 +19350 +18101 +18040 +17877 +16113 +15004 +14441 +12265 +12174 +10215 +10180 +4567 +6100 +5061 +4004 +4005 +8022 +9898 +7999 +1271 +1199 +3003 +1122 +2323 +4224 +2022 +617 +777 +417 +714 +6346 +981 +722 +1009 +4998 +70 +1076 +5999 +10082 +765 +301 +524 +668 +2041 +6009 +1417 +1434 +259 +44443 +1984 +2068 +7004 +1007 +4343 +416 +2038 +6006 +109 +4125 +1461 +9103 +911 +726 +1010 +2046 +2035 +7201 +687 +2013 +481 +125 +6669 +6668 +903 +1455 +683 +1011 +2043 +2047 +256 +9929 +5998 +406 +31337 +44442 +783 +843 +2042 +2045 +4040 +6060 +6051 +1145 +3916 +9443 +9444 +1875 +7272 +4252 +4200 +7024 +1556 +13724 +1141 +1233 +8765 +1137 +3963 +5938 +9191 +3808 +8686 +3981 +2710 +3852 +3849 +3944 +3853 +9988 +1163 +4164 +3820 +6481 +3731 +5081 +40000 +8097 +4555 +3863 +1287 +4430 +7744 +7913 +1166 +1164 +1165 +8019 +10160 +4658 +7878 +3304 +3307 +1259 +1092 +7278 +3872 +10008 +7725 +3410 +1971 +3697 +3859 +3514 +4949 +4147 +7900 +5353 +3931 +8675 +1277 +3957 +1213 +2382 +6600 +3700 +3007 +4080 +1113 +3969 +1132 +1309 +3848 +7281 +3907 +3972 +3968 +1126 +5223 +1217 +3870 +3941 +8293 +1719 +1300 +2099 +6068 +3013 +3050 +1174 +3684 +2170 +3792 +1216 +5151 +7123 +7080 +22222 +4143 +5868 +8889 +12006 +1121 +3119 +8015 +10023 +3824 +1154 +20002 +3888 +4009 +5063 +3376 +1185 +1198 +1192 +1972 +1130 +1149 +4096 +6500 +8294 +3990 +3993 +8016 +5242 +3846 +3929 +1187 +5074 +5909 +8766 +5905 +1102 +2800 +9941 +9914 +9815 +9673 +9643 +9621 +9501 +9409 +9198 +9197 +9098 +8996 +8987 +8877 +8676 +8648 +8540 +8481 +8385 +8189 +8098 +8095 +8050 +7929 +7770 +7749 +7438 +7241 +7051 +7050 +6896 +6732 +6711 +65310 +6520 +6504 +6247 +6203 +61613 +60642 +60146 +60123 +5981 +5940 +59202 +59201 +59200 +5918 +5914 +59110 +5899 +58838 +5869 +58632 +58630 +5823 +5818 +5812 +5807 +58002 +58001 +57665 +55576 +55020 +53535 +5339 +53314 +53313 +53211 +52853 +52851 +52850 +52849 +52847 +5279 +52735 +52710 +52660 +5212 +51413 +51191 +5040 +50050 +49401 +49236 +49195 +49186 +49171 +49168 +49164 +4875 +47544 +46996 +46200 +44709 +41523 +41064 +40811 +3994 +39659 +39376 +39136 +38188 +38185 +37839 +35513 +33554 +33453 +32835 +32822 +32816 +32803 +32792 +32791 +30704 +30005 +29831 +29672 +28211 +27357 +26470 +23796 +23052 +2196 +21792 +19900 +18264 +18018 +17595 +16851 +16800 +16705 +15402 +15001 +12452 +12380 +12262 +12215 +12059 +12021 +10873 +10058 +10034 +10022 +10011 +2910 +1594 +1658 +1583 +3162 +2920 +1812 +26000 +2366 +4600 +1688 +1322 +2557 +1095 +1839 +2288 +1123 +5968 +9600 +1244 +1641 +2200 +1105 +6550 +5501 +1328 +2968 +1805 +1914 +1974 +31727 +3400 +1301 +1147 +1721 +1236 +2501 +2012 +6222 +1220 +1109 +1347 +502 +701 +2232 +2241 +4559 +710 +10005 +5680 +623 +913 +1103 +780 +930 +803 +725 +639 +540 +102 +5010 +1222 +953 +8118 +9992 +1270 +27 +123 +86 +447 +1158 +442 +18000 +419 +931 +874 +856 +250 +475 +2044 +441 +210 +6008 +7003 +5803 +1008 +556 +6103 +829 +3299 +55 +713 +1550 +709 +2628 +223 +3025 +87 +57 +10083 +5520 +980 +251 +1013 +9152 +1212 +2433 +1516 +333 +2011 +748 +1350 +1526 +7010 +1241 +127 +157 +220 +1351 +2067 +684 +77 +4333 +674 +943 +904 +840 +825 +792 +732 +1020 +1006 +657 +557 +610 +1547 +523 +996 +2025 +602 +3456 +862 +600 +2903 +257 +1522 +1353 +6662 +998 +660 +729 +730 +731 +782 +1357 +3632 +3399 +6050 +2201 +971 +969 +905 +846 +839 +823 +822 +795 +790 +778 +757 +659 +225 +1015 +1014 +1012 +655 +786 +6017 +6670 +690 +388 +44334 +754 +5011 +98 +411 +1525 +3999 +740 +12346 +802 +1337 +1127 +2112 +1414 +2600 +621 +606 +59 +928 +924 +922 +921 +918 +878 +864 +859 +806 +805 +728 +252 +1005 +1004 +641 +758 +669 +38037 +715 +1413 +2104 +1229 +3817 +6063 +6062 +6055 +6052 +6030 +6021 +6015 +6010 +3220 +6115 +3940 +2340 +8006 +4141 +3810 +1565 +3511 +33000 +2723 +9202 +4036 +4035 +2312 +3652 +3280 +4243 +4298 +4297 +4294 +4262 +4234 +4220 +4206 +22555 +9300 +7121 +1927 +4433 +5070 +2148 +1168 +9979 +7998 +4414 +1823 +3653 +1223 +8201 +4876 +3240 +2644 +4020 +2436 +3906 +4375 +4024 +5581 +5580 +9694 +6251 +7345 +7325 +7320 +7300 +3121 +5473 +5475 +3600 +3943 +4912 +2142 +1976 +1975 +5202 +5201 +4016 +5111 +9911 +10006 +3923 +3930 +1221 +2973 +3909 +5814 +3080 +4158 +3526 +1911 +5066 +2711 +2187 +3788 +3796 +3922 +2292 +16161 +4881 +3979 +3670 +4174 +3102 +3483 +2631 +1750 +3897 +7500 +5553 +5554 +9875 +4570 +3860 +3712 +8052 +2083 +8883 +2271 +4606 +1208 +3319 +3935 +3430 +1215 +3962 +3368 +3964 +1128 +5557 +4010 +9400 +1605 +3291 +7400 +5005 +1699 +1195 +5053 +3813 +1712 +3002 +3765 +3806 +43000 +2371 +3532 +3799 +3790 +3599 +3850 +4355 +4358 +4357 +4356 +5433 +3928 +4713 +4374 +3961 +9022 +3911 +3396 +7628 +3200 +1753 +3967 +2505 +5133 +3658 +8471 +1314 +2558 +6161 +4025 +3089 +9021 +30001 +8472 +5014 +9990 +1159 +1157 +1308 +5723 +3443 +4161 +1135 +9211 +9210 +4090 +7789 +6619 +9628 +12121 +4454 +3680 +3167 +3902 +3901 +3890 +3842 +16900 +4700 +4687 +8980 +1196 +4407 +3520 +3812 +5012 +10115 +1615 +2902 +4118 +2706 +2095 +2096 +3363 +5137 +3795 +8005 +10007 +3515 +8003 +3847 +3503 +5252 +27017 +2197 +4120 +1180 +5722 +1134 +1883 +1249 +3311 +27350 +3837 +2804 +4558 +4190 +2463 +1204 +4056 +1184 +19333 +9333 +3913 +3672 +4342 +4877 +3586 +8282 +1861 +1752 +9592 +1701 +6085 +2081 +4058 +2115 +8900 +4328 +2958 +2957 +7071 +3899 +2531 +2691 +5052 +1638 +3419 +2551 +5908 +4029 +3603 +1336 +2082 +1143 +3602 +1176 +4100 +3486 +6077 +4800 +2062 +1918 +12001 +12002 +9084 +7072 +1156 +2313 +3952 +4999 +5023 +2069 +28017 +27019 +27018 +3439 +6324 +1188 +1125 +3908 +7501 +8232 +1722 +2988 +10500 +1136 +1162 +10020 +22128 +1211 +3530 +12009 +9005 +3057 +3956 +4325 +1191 +3519 +5235 +1144 +4745 +1901 +1807 +2425 +3210 +32767 +5015 +5013 +3622 +4039 +10101 +5233 +5152 +3983 +3982 +9616 +4369 +3728 +3621 +2291 +5114 +7101 +1315 +2087 +5234 +1635 +3263 +4121 +4602 +2224 +3949 +9131 +3310 +3937 +2253 +3882 +3831 +2376 +2375 +3876 +3362 +3663 +3334 +47624 +1825 +4302 +5721 +1279 +2606 +1173 +22125 +17500 +12005 +6113 +1973 +3793 +3637 +8954 +3742 +9667 +41795 +41794 +4300 +8445 +12865 +3365 +4665 +3190 +3577 +3823 +2261 +2262 +2812 +1190 +22350 +3374 +4135 +2598 +2567 +1167 +8470 +10443 +8116 +3830 +8880 +2734 +3505 +3388 +3669 +1871 +8025 +1958 +3681 +3014 +8999 +4415 +3414 +4101 +6503 +9700 +3683 +1150 +18333 +4376 +3991 +3989 +3992 +2302 +3415 +1179 +3946 +2203 +4192 +4418 +2712 +25565 +4065 +5820 +3915 +2080 +3103 +2265 +8202 +2304 +8060 +4119 +4401 +1560 +3904 +4534 +1835 +1116 +8023 +8474 +3879 +4087 +4112 +6350 +9950 +3506 +3948 +3825 +2325 +1800 +1153 +6379 +3839 +4689 +47806 +5912 +3975 +3980 +4113 +2847 +2070 +3425 +6628 +3997 +3513 +3656 +2335 +1182 +1954 +3996 +4599 +2391 +3479 +5021 +5020 +1558 +1924 +4545 +2991 +6065 +1290 +1559 +1317 +5423 +1707 +5055 +9975 +9971 +9919 +9915 +9912 +9910 +9908 +9901 +9844 +9830 +9826 +9825 +9823 +9814 +9812 +9777 +9745 +9683 +9680 +9679 +9674 +9665 +9661 +9654 +9648 +9620 +9619 +9613 +9583 +9527 +9513 +9493 +9478 +9464 +9454 +9364 +9351 +9183 +9170 +9133 +9130 +9128 +9125 +9065 +9061 +9044 +9037 +9013 +9004 +8925 +8898 +8887 +8882 +8879 +8878 +8865 +8843 +8801 +8798 +8790 +8772 +8756 +8752 +8736 +8680 +8673 +8658 +8655 +8644 +8640 +8621 +8601 +8562 +8539 +8531 +8530 +8515 +8484 +8479 +8477 +8455 +8454 +8453 +8452 +8451 +8409 +8339 +8308 +8295 +8273 +8268 +8255 +8248 +8245 +8144 +8133 +8110 +8092 +8064 +8037 +8029 +8018 +8014 +7975 +7895 +7854 +7853 +7852 +7830 +7813 +7788 +7780 +7772 +7771 +7688 +7685 +7654 +7637 +7600 +7555 +7553 +7456 +7451 +7231 +7218 +7184 +7119 +7104 +7102 +7092 +7068 +7067 +7043 +7033 +6973 +6972 +6956 +6942 +6922 +6920 +6897 +6877 +6780 +6734 +6725 +6710 +6709 +6650 +6647 +6644 +6606 +65514 +65488 +6535 +65311 +65048 +64890 +64727 +64726 +64551 +64507 +64438 +64320 +6412 +64127 +64080 +63803 +63675 +6349 +63423 +6323 +63156 +6310 +63105 +6309 +62866 +6274 +6273 +62674 +6259 +62570 +62519 +6250 +62312 +62188 +62080 +62042 +62006 +61942 +61851 +61827 +61734 +61722 +61669 +61617 +61616 +61516 +61473 +61402 +6126 +6120 +61170 +61169 +61159 +60989 +6091 +6090 +60794 +60789 +60783 +60782 +60753 +60743 +60728 +60713 +6067 +60628 +60621 +60612 +60579 +60544 +60504 +60492 +60485 +60403 +60401 +60377 +60279 +60243 +60227 +60177 +60111 +60086 +60055 +60003 +60002 +60000 +59987 +59841 +59829 +59810 +59778 +5975 +5974 +5971 +59684 +5966 +5958 +59565 +5954 +5953 +59525 +59510 +59509 +59504 +5949 +59499 +5948 +5945 +5939 +5936 +5934 +59340 +5931 +5927 +5926 +5924 +5923 +59239 +5921 +5920 +59191 +5917 +59160 +59149 +59122 +59107 +59087 +58991 +58970 +58908 +5888 +5887 +5881 +5878 +5875 +5874 +58721 +5871 +58699 +58634 +58622 +58610 +5860 +5858 +58570 +58562 +5854 +5853 +5852 +5849 +58498 +5848 +58468 +5845 +58456 +58446 +58430 +5840 +5839 +5838 +58374 +5836 +5834 +5831 +58310 +58305 +5827 +5826 +58252 +5824 +5821 +5817 +58164 +58109 +58107 +5808 +58072 +5806 +5804 +57999 +57988 +57928 +57923 +57896 +57891 +57733 +57730 +57702 +57681 +57678 +57576 +57479 +57398 +57387 +5737 +57352 +57350 +5734 +57347 +57335 +5732 +57325 +57123 +5711 +57103 +57020 +56975 +56973 +56827 +56822 +56810 +56725 +56723 +56681 +5667 +56668 +5665 +56591 +56535 +56507 +56293 +56259 +5622 +5621 +5620 +5612 +5611 +56055 +56016 +55948 +55910 +55907 +55901 +55781 +55773 +55758 +55721 +55684 +55652 +55635 +55579 +55569 +55568 +55556 +5552 +55527 +55479 +55426 +55400 +55382 +55350 +55312 +55227 +55187 +55183 +55000 +54991 +54987 +54907 +54873 +54741 +54722 +54688 +54658 +54605 +5458 +5457 +54551 +54514 +5444 +5442 +5441 +54323 +54321 +54276 +54263 +54235 +54127 +54101 +54075 +53958 +53910 +53852 +53827 +53782 +5377 +53742 +5370 +53690 +53656 +53639 +53633 +53491 +5347 +53469 +53460 +53370 +53361 +53319 +53240 +53212 +53189 +53178 +53085 +52948 +5291 +52893 +52675 +52665 +5261 +5259 +52573 +52506 +52477 +52391 +52262 +52237 +52230 +52226 +52225 +5219 +52173 +52071 +52046 +52025 +52003 +52002 +52001 +52000 +51965 +51961 +51909 +51906 +51809 +51800 +51772 +51771 +51658 +51582 +51515 +51488 +51485 +51484 +5147 +51460 +51423 +51366 +51351 +51343 +51300 +5125 +51240 +51235 +51234 +51233 +5122 +5121 +51139 +51118 +51067 +51037 +51020 +51011 +50997 +5098 +5096 +5095 +50945 +5090 +50903 +5088 +50887 +50854 +50849 +50836 +50835 +50834 +50833 +50831 +50815 +50809 +50787 +50733 +50692 +50585 +50577 +50576 +50545 +50529 +50513 +50356 +50277 +50258 +50246 +50224 +50205 +50202 +50198 +50189 +5017 +5016 +50101 +50040 +50019 +50016 +49927 +49803 +49765 +49762 +49751 +49678 +49603 +49597 +49522 +49521 +49520 +49519 +49500 +49498 +49452 +49398 +49372 +49352 +4931 +49302 +49275 +49241 +49235 +49232 +49228 +49216 +49213 +49211 +49204 +49203 +49202 +49201 +49197 +49196 +49191 +49190 +49189 +49179 +49173 +49172 +49170 +49169 +49166 +49132 +49048 +4903 +49002 +48973 +48967 +48966 +48925 +48813 +48783 +48682 +48648 +48631 +4860 +4859 +48434 +48356 +4819 +48167 +48153 +48127 +48083 +48067 +48009 +47969 +47966 +4793 +47860 +47858 +47850 +4778 +47777 +4771 +4770 +47700 +4767 +47634 +4760 +47595 +47581 +47567 +47448 +47372 +47348 +47267 +47197 +4712 +47119 +47029 +47012 +46992 +46813 +46593 +4649 +4644 +46436 +46418 +46372 +46310 +46182 +46171 +46115 +4609 +46069 +46034 +45960 +45864 +45777 +45697 +45624 +45602 +45463 +45438 +45413 +4530 +45226 +45220 +4517 +4516 +45164 +45136 +45050 +45038 +44981 +44965 +4476 +4471 +44711 +44704 +4464 +44628 +44616 +44541 +44505 +44479 +44431 +44410 +44380 +44200 +44119 +44101 +44004 +4388 +43868 +4384 +43823 +43734 +43690 +43654 +43425 +43242 +43231 +43212 +43143 +43139 +43103 +43027 +43018 +43002 +42990 +42906 +42735 +42685 +42679 +42675 +42632 +42590 +42575 +42560 +42559 +42452 +42449 +42322 +42276 +42251 +42158 +42127 +42035 +42001 +41808 +41773 +41632 +41551 +41442 +41398 +41348 +41345 +41342 +41318 +41281 +41250 +41142 +41123 +40951 +40834 +40812 +40754 +40732 +40712 +40628 +40614 +40513 +40489 +40457 +40400 +40393 +40306 +40011 +40005 +40003 +40002 +40001 +39917 +39895 +39883 +39869 +39795 +39774 +39763 +39732 +39630 +39489 +39482 +39433 +39380 +39293 +39265 +39117 +39067 +38936 +38805 +38780 +38764 +38761 +38570 +38561 +38546 +38481 +38446 +38358 +38331 +38313 +38270 +38224 +38205 +38194 +38029 +37855 +37789 +37777 +37674 +37647 +37614 +37607 +37522 +37393 +37218 +37185 +37174 +37151 +37121 +36983 +36962 +36950 +36914 +36824 +36823 +36748 +36710 +36694 +36677 +36659 +36552 +36530 +36508 +36436 +36368 +36275 +36256 +36105 +36104 +36046 +35986 +35929 +35906 +35901 +35900 +35879 +35731 +35593 +35553 +35506 +35401 +35393 +35392 +35349 +35272 +35217 +35131 +35116 +35050 +35033 +34875 +34833 +34783 +34765 +34728 +34683 +34510 +34507 +34401 +34381 +34341 +34317 +34189 +34096 +34036 +34021 +33895 +33889 +33882 +33879 +33841 +33605 +33604 +33550 +33523 +33522 +33444 +33395 +33367 +33337 +33335 +33327 +33277 +33203 +33200 +33192 +33175 +33124 +33087 +33070 +33017 +33011 +32976 +32961 +32960 +32944 +32932 +32911 +32910 +32908 +32905 +32904 +32898 +32897 +32888 +32871 +32869 +32868 +32858 +32842 +32837 +32820 +32815 +32814 +32807 +32799 +32798 +32797 +32790 +32789 +32788 +32765 +32764 +32261 +32260 +32219 +32200 +32102 +32088 +32031 +32022 +32006 +31728 +31657 +31522 +31438 +31386 +31339 +31072 +31058 +31033 +30896 +30705 +30659 +30644 +30599 +30519 +30299 +30195 +30087 +29810 +29507 +29243 +29152 +29045 +28967 +28924 +28851 +28850 +28717 +28567 +28374 +28142 +28114 +27770 +27537 +27521 +27372 +27351 +27316 +27204 +27087 +27075 +27074 +27055 +27016 +27015 +26972 +26669 +26417 +26340 +26007 +26001 +25847 +25717 +25703 +25486 +25473 +25445 +25327 +25288 +25262 +25260 +25174 +24999 +24616 +24552 +24416 +24392 +24218 +23953 +23887 +23723 +23451 +23430 +23382 +23342 +23296 +23270 +23228 +23219 +23040 +23017 +22969 +22959 +22882 +22769 +22727 +22719 +22711 +22563 +22341 +22290 +22223 +22200 +22177 +22100 +22063 +22022 +21915 +21891 +21728 +21634 +21631 +21473 +21078 +21011 +20990 +20940 +20934 +20883 +20734 +20473 +20280 +20228 +20227 +20226 +20225 +20224 +20223 +20180 +20179 +20147 +20127 +20125 +20118 +20111 +20106 +20102 +20089 +20085 +20080 +20076 +20052 +20039 +20032 +20021 +20017 +20011 +19996 +19995 +19852 +19715 +19634 +19612 +19501 +19464 +19403 +19353 +19201 +19200 +19130 +19010 +18962 +18910 +18887 +18874 +18669 +18569 +18517 +18505 +18439 +18380 +18337 +18336 +18231 +18148 +18080 +18015 +18012 +17997 +17985 +17969 +17867 +17860 +17802 +17801 +17715 +17702 +17701 +17700 +17413 +17409 +17255 +17251 +17129 +17089 +17070 +17017 +17016 +16901 +16845 +16797 +16725 +16724 +16723 +16464 +16372 +16349 +16297 +16286 +16283 +16273 +16270 +16048 +15915 +15758 +15730 +15722 +15677 +15670 +15646 +15645 +15631 +15550 +15448 +15344 +15317 +15275 +15191 +15190 +15145 +15050 +15005 +14916 +14891 +14827 +14733 +14693 +14545 +14534 +14444 +14443 +14418 +14254 +14237 +14218 +14147 +13899 +13846 +13784 +13766 +13730 +13723 +13695 +13580 +13502 +13359 +13340 +13318 +13306 +13265 +13264 +13261 +13250 +13229 +13194 +13193 +13192 +13188 +13167 +13149 +13142 +13140 +13132 +13130 +13093 +13017 +12962 +12955 +12892 +12891 +12766 +12702 +12699 +12414 +12340 +12296 +12275 +12271 +12251 +12243 +12240 +12225 +12192 +12171 +12156 +12146 +12137 +12132 +12097 +12096 +12090 +12080 +12077 +12034 +12031 +12019 +11940 +11863 +11862 +11813 +11735 +11697 +11552 +11401 +11296 +11288 +11250 +11224 +11200 +11180 +11100 +11089 +11033 +11032 +11031 +11026 +11019 +11007 +11003 +10900 +10878 +10852 +10842 +10754 +10699 +10602 +10601 +10567 +10565 +10556 +10555 +10554 +10553 +10552 +10551 +10550 +10535 +10529 +10509 +10494 +10414 +10387 +10357 +10347 +10338 +10280 +10255 +10246 +10245 +10238 +10093 +10064 +10045 +10042 +10035 +10019 +10018 +1327 +2330 +2580 +2700 +1584 +9020 +3281 +2439 +1250 +14001 +1607 +1736 +1330 +2270 +2728 +2888 +3803 +5250 +1645 +1303 +3636 +1251 +1243 +1291 +1297 +1200 +1811 +4442 +1118 +8401 +2101 +2889 +1694 +1730 +1912 +29015 +28015 +1745 +2250 +1306 +2997 +2449 +1262 +4007 +1101 +1268 +1735 +1858 +1264 +1711 +3118 +4601 +1321 +1598 +1305 +1632 +9995 +1307 +1981 +2532 +1808 +2435 +1194 +1622 +1239 +1799 +2882 +1683 +3063 +3062 +1340 +4447 +1806 +6888 +2438 +1261 +5969 +9343 +2583 +2031 +3798 +2269 +20001 +2622 +11001 +1207 +2850 +21201 +2908 +3936 +3023 +2280 +2623 +7099 +2372 +1318 +1339 +1276 +11000 +48619 +3497 +1209 +1331 +1240 +3856 +2987 +2326 +25001 +25000 +1792 +3919 +1299 +2984 +1715 +1703 +1677 +2086 +1708 +1228 +3787 +5502 +1620 +1316 +1569 +1210 +1691 +1282 +2124 +1791 +2150 +9909 +4022 +3868 +1324 +2584 +2300 +9287 +2806 +1566 +1713 +1592 +3749 +1302 +1709 +3485 +2418 +2472 +24554 +3146 +2134 +2898 +9161 +9160 +2930 +1319 +5672 +3811 +2456 +2901 +6579 +2550 +8403 +31416 +22273 +7005 +66 +32786 +32787 +706 +914 +635 +6105 +400 +47 +830 +4008 +5977 +1989 +1444 +3985 +678 +27001 +591 +642 +446 +1441 +54320 +11 +769 +983 +979 +973 +967 +965 +961 +942 +935 +926 +925 +863 +858 +844 +834 +817 +815 +811 +809 +789 +779 +743 +1019 +1507 +1492 +509 +762 +5632 +578 +1495 +5308 +52 +219 +525 +1420 +665 +620 +3064 +3045 +653 +158 +716 +861 +9991 +3049 +1366 +1364 +833 +91 +1680 +3398 +750 +615 +603 +6110 +101 +989 +27010 +510 +810 +1139 +4199 +76 +847 +649 +707 +68 +449 +664 +75 +104 +629 +1652 +682 +577 +985 +984 +974 +958 +952 +949 +946 +923 +916 +899 +897 +894 +889 +835 +824 +814 +807 +804 +798 +733 +727 +237 +12 +10 +501 +122 +440 +771 +1663 +828 +860 +695 +634 +538 +1359 +1358 +1517 +1370 +3900 +492 +268 +27374 +605 +8076 +1651 +1178 +6401 +761 +5145 +50 +2018 +1349 +2014 +7597 +2120 +1445 +1402 +1465 +9104 +627 +4660 +7273 +950 +1384 +1388 +760 +92 +831 +5978 +4557 +45 +112 +456 +1214 +3086 +702 +6665 +1404 +651 +5300 +6347 +5400 +1389 +647 +448 +1356 +5232 +1484 +450 +1991 +1988 +1523 +1400 +1399 +221 +1385 +5191 +1346 +2024 +2430 +988 +962 +948 +945 +941 +938 +936 +929 +927 +919 +906 +883 +881 +875 +872 +870 +866 +855 +851 +850 +841 +836 +826 +820 +819 +816 +813 +791 +745 +736 +735 +724 +719 +343 +334 +300 +28 +249 +230 +16 +1018 +1016 +658 +1474 +696 +630 +663 +2307 +1552 +609 +741 +353 +638 +1551 +661 +491 +640 +507 +673 +632 +1354 +9105 +6143 +676 +214 +14141 +182 +69 +27665 +1475 +97 +633 +560 +799 +7009 +2015 +628 +751 +4480 +1403 +8123 +1527 +723 +1466 +1486 +1650 +991 +832 +137 +1348 +685 +1762 +6701 +994 +4500 +194 +180 +1539 +1379 +51 +886 +2064 +1405 +1435 +11371 +1401 +1369 +402 +103 +1372 +704 +854 +8892 +47557 +624 +1387 +3397 +1996 +1995 +1997 +18182 +18184 +3264 +3292 +13720 +9107 +9106 +201 +1381 +35 +6588 +5530 +3141 +670 +970 +968 +964 +963 +960 +959 +951 +947 +944 +939 +933 +909 +895 +891 +879 +869 +868 +867 +837 +821 +812 +797 +796 +794 +788 +756 +734 +721 +718 +708 +703 +60 +40 +253 +231 +14 +1017 +1003 +656 +975 +2026 +1497 +553 +511 +611 +689 +1668 +1664 +15 +561 +997 +505 +1496 +637 +213 +1412 +1515 +692 +694 +681 +680 +644 +675 +1467 +454 +622 +1476 +1373 +770 +262 +654 +1535 +58 +177 +26208 +677 +1519 +1398 +3457 +401 +412 +493 +13713 +94 +1498 +871 +1390 +6145 +133 +362 +118 +193 +115 +1549 +7008 +608 +1426 +1436 +915 +38 +74 +73 +71 +601 +136 +4144 +129 +16444 +1446 +4132 +308 +1528 +1365 +1393 +1394 +1493 +138 +5997 +397 +29 +31 +44 +2627 +6147 +1510 +568 +350 +2053 +6146 +6544 +1763 +3531 +399 +1537 +1992 +1355 +1454 +261 +887 +200 +1376 +1424 +6111 +1410 +1409 +686 +5301 +5302 +1513 +747 +9051 +1499 +7006 +1439 +1438 +8770 +853 +196 +93 +410 +462 +619 +1529 +1990 +1994 +1986 +1386 +18183 +18181 +6700 +1442 +95 +6400 +1432 +1548 +486 +1422 +114 +1397 +6142 +1827 +626 +422 +688 +206 +202 +204 +1483 +7634 +774 +699 +2023 +776 +672 +1545 +2431 +697 +982 +978 +972 +966 +957 +956 +934 +920 +908 +907 +892 +890 +885 +884 +882 +877 +876 +865 +857 +852 +849 +842 +838 +827 +818 +793 +785 +784 +755 +746 +738 +737 +717 +34 +336 +325 +303 +276 +273 +236 +235 +233 +181 +604 +1362 +712 +1437 +2027 +1368 +1531 +645 +65301 +260 +536 +764 +698 +607 +1667 +1662 +1661 +404 +224 +418 +176 +848 +315 +466 +403 +1456 +1479 +355 +763 +1472 +453 +759 +437 +2432 +120 +415 +1544 +1511 +1538 +346 +173 +54 +56 +265 +1462 +13701 +1518 +1457 +117 +1470 +13715 +13714 +267 +1419 +1418 +1407 +380 +518 +65 +391 +392 +413 +1391 +614 +1408 +162 +108 +4987 +1502 +598 +582 +487 +530 +1509 +72 +4672 +189 +209 +270 +7464 +408 +191 +1459 +5714 +5717 +5713 +564 +767 +583 +1395 +192 +1448 +428 +4133 +1416 +773 +1458 +526 +1363 +742 +1464 +1427 +1482 +569 +571 +6141 +351 +3984 +5490 +2 +13718 +373 +17300 +910 +148 +7326 +271 +423 +1451 +480 +1430 +1429 +781 +383 +2564 +613 +612 +652 +5303 +1383 +128 +19150 +1453 +190 +1505 +1371 +533 +27009 +27007 +27005 +27003 +27002 +744 +1423 +1374 +141 +1440 +1396 +352 +96 +48 +552 +570 +217 +528 +452 +451 +2766 +2108 +132 +1993 +1987 +130 +18187 +216 +3421 +142 +13721 +67 +15151 +364 +1411 +205 +6548 +124 +116 +5193 +258 +485 +599 +149 +1469 +775 +2019 +516 +986 +977 +976 +955 +954 +937 +932 +8 +896 +893 +845 +768 +766 +739 +337 +329 +326 +305 +295 +294 +293 +289 +288 +277 +238 +234 +229 +228 +226 +522 +2028 +150 +572 +596 +420 +460 +1543 +358 +361 +470 +360 +457 +643 +322 +168 +753 +369 +185 +43188 +1541 +1540 +752 +496 +662 +1449 +1480 +1473 +184 +1672 +1671 +1670 +435 +434 +1532 +1360 +174 +472 +1361 +17007 +414 +535 +432 +479 +473 +151 +1542 +438 +1488 +1508 +618 +316 +1367 +439 +284 +542 +370 +2016 +248 +1491 +44123 +41230 +7173 +5670 +18136 +3925 +7088 +1425 +17755 +17756 +4072 +5841 +2102 +4123 +2989 +10051 +10050 +31029 +3726 +5243 +9978 +9925 +6061 +6058 +6057 +6056 +6054 +6053 +6049 +6048 +6047 +6046 +6045 +6044 +6043 +6042 +6041 +6040 +6039 +6038 +6037 +6036 +6035 +6034 +6033 +6032 +6031 +6029 +6028 +6027 +6026 +6024 +6023 +6022 +6020 +6019 +6018 +6016 +6014 +6013 +6012 +6011 +36462 +5793 +3423 +3424 +4095 +3646 +3510 +3722 +2459 +3651 +14500 +3865 +15345 +3763 +38422 +3877 +9092 +5344 +3974 +2341 +6116 +2157 +165 +6936 +8041 +4888 +4889 +3074 +2165 +4389 +5770 +5769 +16619 +11876 +11877 +3741 +3633 +3840 +3717 +3716 +3590 +2805 +4537 +9762 +5007 +5006 +5358 +4879 +6114 +4185 +2784 +3724 +2596 +2595 +4417 +4845 +22321 +22289 +3219 +1338 +36411 +3861 +5166 +3674 +1785 +534 +6602 +47001 +5363 +8912 +2231 +5747 +5748 +11208 +7236 +4049 +4050 +22347 +63 +3233 +3359 +8908 +4177 +48050 +3111 +3427 +5321 +5320 +3702 +2907 +8991 +8990 +2054 +4847 +9802 +9800 +4368 +5990 +3563 +5744 +5743 +12321 +12322 +9206 +9204 +9205 +9201 +9203 +2949 +2948 +6626 +37472 +8199 +4145 +3482 +2216 +13708 +3786 +3375 +7566 +2539 +2387 +3317 +2410 +2255 +3883 +4299 +4296 +4295 +4293 +4292 +4291 +4290 +4289 +4288 +4287 +4286 +4285 +4284 +4283 +4282 +4281 +4280 +4278 +4277 +4276 +4275 +4274 +4273 +4272 +4271 +4270 +4269 +4268 +4267 +4266 +4265 +4264 +4263 +4261 +4260 +4259 +4258 +4257 +4256 +4255 +4254 +4253 +4251 +4250 +4249 +4248 +4247 +4246 +4245 +4244 +4241 +4240 +4239 +4238 +4237 +4236 +4235 +4233 +4232 +4231 +4230 +4229 +4228 +4227 +4226 +4225 +4223 +4222 +4221 +4219 +4218 +4217 +4216 +4215 +4214 +4213 +4212 +4211 +4210 +4209 +4208 +4207 +4205 +4204 +4203 +4202 +4201 +2530 +5164 +28200 +3845 +3541 +4052 +21590 +1796 +25793 +8699 +8182 +4991 +2474 +5780 +3676 +24249 +1631 +6672 +6673 +3601 +5046 +3509 +1852 +2386 +8473 +7802 +4789 +3555 +12013 +12012 +3752 +3245 +3231 +16666 +6678 +17184 +9086 +9598 +3073 +2074 +1956 +2610 +3738 +2994 +2993 +2802 +1885 +14149 +13786 +10100 +9284 +14150 +10107 +4032 +2821 +3207 +14154 +24323 +2771 +5646 +2426 +18668 +2554 +4188 +3654 +8034 +5675 +15118 +4031 +2529 +2248 +1142 +19194 +433 +3534 +3664 +2537 +519 +2655 +4184 +1506 +3098 +7887 +37654 +1979 +9629 +2357 +1889 +3314 +3313 +4867 +2696 +3217 +6306 +1189 +5281 +8953 +1910 +13894 +372 +3720 +1382 +2542 +3584 +4034 +145 +27999 +3791 +21800 +2670 +3492 +24678 +34249 +39681 +1846 +5197 +5462 +5463 +2862 +2977 +2978 +3468 +2675 +3474 +4422 +12753 +13709 +2573 +3012 +4307 +4725 +3346 +3686 +4070 +9555 +4711 +4323 +4322 +10200 +7727 +3608 +3959 +2405 +3858 +3857 +24322 +6118 +4176 +6442 +8937 +17224 +17225 +7234 +33434 +1906 +22351 +2158 +5153 +3885 +24465 +3040 +20167 +8066 +474 +2739 +3308 +590 +3309 +7902 +7901 +7903 +20046 +5582 +5583 +7872 +13716 +13717 +13705 +6252 +2915 +1965 +3459 +3160 +3754 +3243 +10261 +7932 +7933 +5450 +11971 +379 +7548 +1832 +28080 +3805 +16789 +8320 +8321 +4423 +2296 +7359 +7358 +7357 +7356 +7355 +7354 +7353 +7352 +7351 +7350 +7349 +7348 +7347 +7346 +7344 +7343 +7342 +7341 +7340 +7339 +7338 +7337 +7336 +7335 +7334 +7333 +7332 +7331 +7330 +7329 +7328 +7327 +7324 +7323 +7322 +7321 +7319 +7318 +7317 +7316 +7315 +7314 +7313 +7312 +7311 +7310 +7309 +7308 +7307 +7306 +7305 +7304 +7303 +7302 +7301 +8140 +5196 +5195 +6130 +5474 +5471 +5472 +5470 +4146 +3713 +5048 +31457 +7631 +3544 +41121 +11600 +3696 +3549 +1380 +22951 +22800 +3521 +2060 +6083 +9668 +3552 +1814 +1977 +2576 +2729 +24680 +13710 +13712 +25900 +2403 +2402 +2470 +5203 +3579 +2306 +1450 +7015 +7012 +7011 +22763 +2156 +2493 +4019 +4018 +4017 +4015 +2392 +3175 +32249 +1627 +10104 +2609 +5406 +3251 +4094 +3241 +6514 +6418 +3734 +2679 +4953 +5008 +2880 +8243 +8280 +26133 +8555 +5629 +3547 +5639 +5638 +5637 +5115 +3723 +4950 +3895 +3894 +3491 +3318 +6419 +3185 +243 +3212 +9536 +1925 +11171 +8404 +8405 +8989 +6787 +6483 +3867 +3866 +1860 +1870 +5306 +3816 +7588 +6786 +2084 +11165 +11161 +11163 +11162 +11164 +3708 +4850 +7677 +16959 +247 +3478 +5349 +3854 +5397 +7411 +9612 +11173 +9293 +5027 +5026 +5705 +8778 +527 +1312 +8808 +6144 +4157 +4156 +3249 +7471 +3615 +5777 +2154 +45966 +17235 +3018 +38800 +2737 +156 +3807 +2876 +1759 +7981 +3606 +3647 +3438 +4683 +9306 +9312 +7016 +33334 +3413 +3834 +3835 +2440 +6121 +8668 +2568 +17185 +7982 +2290 +2569 +2863 +1964 +4738 +2132 +17777 +16162 +6551 +3230 +4538 +3884 +9282 +9281 +4882 +5146 +580 +1967 +2659 +2409 +5416 +2657 +3380 +5417 +2658 +5161 +5162 +10162 +10161 +33656 +7560 +2599 +2704 +2703 +4170 +7734 +9522 +3158 +4426 +4786 +2721 +1608 +3516 +4988 +4408 +1847 +36423 +2826 +2827 +3556 +8111 +6456 +6455 +3874 +3611 +2629 +2630 +166 +5059 +3110 +1733 +40404 +2257 +2278 +4750 +4303 +3688 +4751 +5794 +4752 +7626 +16950 +3273 +3896 +3635 +1959 +4753 +2857 +4163 +1659 +2905 +2904 +2733 +4936 +5032 +3048 +29000 +28240 +2320 +4742 +22335 +22333 +5043 +4105 +1257 +3841 +43210 +4366 +5163 +11106 +5434 +6444 +6445 +5634 +5636 +5635 +6343 +4546 +3242 +5568 +4057 +24666 +21221 +6488 +6484 +6486 +6485 +6487 +6443 +6480 +6489 +7690 +2603 +4787 +2367 +9212 +9213 +5445 +45824 +8351 +13711 +4076 +5099 +2316 +3588 +5093 +9450 +8056 +8055 +8054 +8059 +8058 +8057 +8053 +3090 +3255 +2254 +2479 +2477 +2478 +4194 +3496 +3495 +2089 +38865 +9026 +9025 +9024 +9023 +3480 +1905 +3550 +7801 +2189 +5361 +32635 +3782 +3432 +3978 +6629 +3143 +7784 +2342 +2309 +2705 +2310 +2384 +6315 +5343 +9899 +5168 +5167 +3927 +266 +2577 +5307 +3838 +19007 +7708 +37475 +7701 +5435 +3499 +2719 +3352 +25576 +3942 +1644 +3755 +5574 +5573 +7542 +9310 +1129 +4079 +3038 +8768 +4033 +9401 +9402 +20012 +20013 +30832 +1606 +5410 +5422 +5409 +9801 +7743 +14034 +14033 +4952 +21801 +3452 +2760 +3153 +23272 +2578 +5156 +8554 +7401 +3771 +3138 +3137 +3500 +6900 +363 +3455 +1698 +13217 +2752 +3864 +10201 +6568 +2377 +3677 +520 +2258 +4124 +8051 +2223 +3194 +4041 +48653 +8270 +5693 +25471 +2416 +5994 +9208 +7810 +7870 +2249 +7473 +4664 +4590 +2777 +2776 +2057 +6148 +3296 +4410 +4684 +8230 +5842 +1431 +12109 +4756 +4336 +324 +323 +3019 +39 +2225 +4733 +30100 +2999 +3422 +107 +1232 +3418 +3537 +5 +8184 +3789 +5231 +4731 +4373 +45045 +12302 +2373 +6084 +16665 +16385 +18635 +18634 +10253 +7227 +3572 +3032 +5786 +2346 +2348 +2347 +2349 +45002 +3553 +43191 +5313 +3707 +3706 +3736 +32811 +1942 +44553 +35001 +35002 +35005 +35006 +35003 +35004 +532 +2214 +5569 +3142 +2332 +3768 +2774 +2773 +6099 +2167 +2714 +2713 +3533 +4037 +2457 +1953 +9345 +21553 +2408 +2736 +2188 +18104 +1813 +469 +1596 +3178 +5430 +5676 +2177 +4841 +5028 +7980 +3166 +3554 +3566 +3843 +5677 +7040 +2589 +8153 +10055 +5464 +2497 +4354 +9222 +5083 +5082 +45825 +2612 +6980 +5689 +6209 +2523 +2490 +2468 +3543 +5543 +7794 +4193 +4951 +3951 +4093 +7747 +7997 +8117 +6140 +2873 +4329 +320 +319 +597 +3453 +4457 +2303 +5360 +4487 +409 +344 +1460 +5716 +5715 +9640 +5798 +7663 +7798 +7797 +4352 +15999 +34962 +34963 +34964 +4749 +8032 +4182 +1283 +1778 +3248 +2722 +2039 +3650 +3133 +2618 +4168 +10631 +1392 +3910 +6716 +47809 +38638 +4690 +9280 +6163 +2315 +3607 +5630 +4455 +4456 +1587 +28001 +5134 +13224 +13223 +5507 +2443 +4150 +8432 +7172 +3710 +9889 +6464 +7787 +6771 +6770 +3055 +2487 +16310 +16311 +3540 +34379 +34378 +2972 +7633 +6355 +188 +2790 +32400 +4351 +3934 +3933 +4659 +1819 +5586 +5863 +17010 +9318 +318 +5318 +2634 +4416 +5078 +3189 +6924 +3010 +15740 +1603 +2787 +4390 +468 +4869 +4868 +3177 +3347 +6124 +2350 +3208 +2520 +2441 +3109 +3557 +281 +1916 +4313 +5312 +4066 +345 +9630 +9631 +6817 +3582 +9279 +9278 +8027 +3587 +4747 +2178 +5112 +3135 +5443 +7880 +1980 +6086 +3254 +4012 +9597 +3253 +2274 +2299 +8444 +6655 +44322 +44321 +5351 +5350 +5172 +4172 +1332 +2256 +8129 +8128 +4097 +8161 +2665 +2664 +6162 +4189 +1333 +3735 +586 +6581 +6582 +4681 +4312 +4989 +7216 +3348 +3095 +6657 +30002 +7237 +3435 +2246 +1675 +31400 +4311 +9559 +6671 +6679 +3034 +40853 +11103 +3274 +3355 +3078 +3075 +3076 +8070 +2484 +2483 +3891 +1571 +1830 +1630 +8997 +8102 +2482 +2481 +5155 +5575 +3718 +22005 +22004 +22003 +22002 +2524 +1829 +2237 +3977 +3976 +3303 +19191 +3433 +5724 +2400 +7629 +6640 +2389 +30999 +2447 +3673 +7430 +7429 +7426 +7431 +7428 +7427 +9390 +4317 +35357 +7728 +8004 +5045 +8688 +1258 +5757 +5729 +5767 +5766 +5755 +5768 +4743 +9008 +9007 +3187 +20014 +4089 +3434 +4840 +4843 +3100 +314 +3154 +9994 +9993 +8767 +4304 +2428 +2199 +2198 +2185 +4428 +4429 +4162 +4395 +2056 +5402 +3340 +3339 +3341 +3338 +7275 +7274 +7277 +7276 +4359 +2077 +8769 +9966 +4732 +3320 +11175 +11174 +11172 +13706 +3523 +429 +2697 +18186 +3442 +3441 +29167 +36602 +7030 +1894 +28000 +126 +4420 +2184 +3780 +49001 +11235 +4128 +8711 +10810 +45001 +5415 +4453 +359 +3266 +36424 +2868 +7724 +396 +2645 +23402 +23400 +23401 +3016 +21010 +5215 +4663 +4803 +2338 +15126 +8433 +5209 +3406 +3405 +5627 +4088 +2210 +2244 +2817 +10111 +10110 +1242 +5299 +2252 +3649 +6421 +6420 +1617 +48001 +48002 +48003 +48005 +48004 +48000 +61 +8061 +4134 +38412 +20048 +7393 +4021 +178 +8457 +550 +2058 +2075 +2076 +3165 +6133 +2614 +2585 +4702 +4701 +2586 +3203 +3204 +4460 +16361 +16367 +16360 +16368 +4159 +170 +2293 +4703 +8981 +3409 +7549 +171 +20049 +1155 +537 +3196 +3195 +2411 +2788 +4127 +6777 +6778 +1879 +5421 +3440 +2128 +21846 +21849 +21847 +21848 +395 +154 +155 +4425 +2328 +3129 +3641 +3640 +1970 +2486 +2485 +6842 +6841 +3149 +3148 +3150 +3151 +1406 +218 +10116 +10114 +2219 +2735 +10117 +10113 +2220 +3725 +5229 +4350 +6513 +4335 +4334 +5681 +1676 +2971 +4409 +3131 +4441 +1612 +1616 +1613 +1614 +13785 +11104 +11105 +3829 +11095 +3507 +3213 +7474 +3886 +4043 +2730 +377 +378 +3024 +2738 +2528 +4844 +4842 +5979 +1888 +2093 +2094 +20034 +2163 +3159 +6317 +4361 +2895 +3753 +2343 +3015 +1790 +3950 +6363 +9286 +9285 +7282 +6446 +2273 +33060 +2388 +9119 +3733 +32801 +4421 +7420 +9903 +6622 +5354 +7742 +2305 +2791 +8115 +3122 +2855 +8276 +2871 +4554 +2171 +2172 +2173 +2174 +7680 +3343 +7392 +3958 +3358 +46 +6634 +8503 +3924 +2488 +10544 +10543 +10541 +10540 +10542 +4691 +8666 +1576 +4986 +6997 +3732 +4688 +7871 +9632 +7869 +2593 +3764 +5237 +4668 +4173 +4667 +8077 +4310 +7606 +5136 +4069 +21554 +7391 +9445 +2180 +3180 +2621 +4551 +3008 +7013 +7014 +5362 +6601 +1512 +5356 +6074 +5726 +5364 +5725 +6076 +6075 +2175 +3132 +5359 +2176 +5022 +4679 +4680 +6509 +2266 +6382 +2230 +6390 +6370 +6360 +393 +2311 +8787 +18 +8786 +47000 +19788 +1960 +9596 +4603 +4151 +4552 +11211 +3569 +4883 +3571 +2944 +2945 +2272 +7720 +5157 +3445 +2427 +2727 +2363 +46999 +2789 +13930 +3232 +2688 +3235 +5598 +3115 +3117 +3116 +3331 +3332 +3302 +3330 +3558 +8809 +3570 +4153 +2591 +4179 +4171 +3276 +5540 +4360 +8448 +4458 +7421 +49000 +7073 +3836 +5282 +8384 +36700 +4686 +269 +9255 +6201 +2544 +2516 +5092 +2243 +4902 +313 +3691 +2453 +4345 +44900 +36444 +36443 +4894 +3747 +3746 +5044 +6471 +3079 +4913 +4741 +10805 +3487 +3157 +3068 +8162 +4083 +4082 +4081 +7026 +1983 +2289 +1629 +1628 +1634 +8101 +6482 +5254 +5058 +4044 +3591 +3592 +1903 +5062 +6087 +2090 +2465 +2466 +6200 +8208 +8207 +8204 +31620 +8205 +8206 +3278 +2145 +2143 +2147 +2146 +3767 +46336 +10933 +4341 +1969 +10809 +12300 +8191 +517 +4670 +7365 +3028 +3027 +3029 +1203 +1886 +11430 +374 +2212 +3407 +2816 +2779 +2815 +2780 +3373 +3739 +3815 +4347 +11796 +3970 +4547 +1764 +2395 +4372 +4432 +9747 +4371 +3360 +3361 +4331 +40023 +27504 +2294 +5253 +7697 +35354 +186 +30260 +4566 +584 +5696 +6623 +6620 +6621 +2502 +3112 +36865 +2918 +4661 +31016 +26262 +26263 +3642 +48048 +5309 +3155 +4166 +27442 +6583 +3215 +3214 +8901 +19020 +4160 +3094 +3093 +3777 +1937 +1938 +1939 +1940 +2097 +1936 +1810 +6244 +6243 +6242 +6241 +4107 +19541 +3529 +3528 +5230 +4327 +5883 +2205 +7095 +3794 +3473 +3472 +7181 +5034 +3627 +8091 +1578 +5673 +5049 +4880 +3258 +2828 +3719 +7478 +7280 +1636 +1637 +3775 +24321 +499 +3205 +1950 +1949 +3226 +8148 +5047 +4075 +17223 +21000 +3504 +3206 +2632 +529 +4073 +32034 +18769 +2527 +4593 +4792 +4791 +7031 +33435 +4740 +4739 +4068 +20202 +4737 +9214 +2215 +3743 +2088 +7410 +5728 +45054 +3614 +8020 +11751 +2202 +6697 +4744 +1884 +3699 +6714 +1611 +7202 +4569 +3508 +24386 +16995 +16994 +1674 +1673 +7128 +4746 +17234 +9215 +4486 +484 +5057 +5056 +7624 +2980 +4109 +49150 +215 +23005 +23004 +23003 +23002 +23001 +23000 +2716 +3560 +5597 +134 +38001 +38000 +4067 +1428 +2480 +5029 +8067 +5069 +3156 +3139 +244 +7675 +7673 +7672 +7674 +2637 +4139 +3783 +3657 +11320 +8615 +585 +48128 +2239 +3596 +2055 +3186 +19000 +5165 +3420 +17220 +17221 +19998 +2404 +2079 +4152 +4604 +25604 +5742 +5741 +4553 +2799 +4801 +4802 +2063 +14143 +14142 +4061 +4062 +4063 +4064 +31948 +31949 +2276 +2275 +1881 +2078 +3660 +3661 +1920 +1919 +9085 +424 +1933 +1934 +9089 +9088 +3667 +3666 +12003 +12004 +3539 +3538 +3267 +25100 +385 +3494 +4594 +4595 +4596 +3898 +9614 +4169 +5674 +2374 +5105 +8313 +44323 +5628 +2570 +2113 +4591 +4592 +5228 +5224 +5227 +2207 +4484 +3037 +2209 +2448 +3101 +382 +381 +3209 +7510 +2206 +2690 +2208 +7738 +5317 +3329 +5316 +3449 +2029 +1985 +10125 +2597 +3634 +8231 +3250 +43438 +4884 +4117 +2467 +4148 +18516 +7397 +22370 +8807 +3921 +4306 +10860 +6440 +3740 +1161 +2641 +7630 +3804 +4197 +11108 +9954 +6791 +3623 +3769 +3036 +5315 +5305 +3542 +5304 +11720 +2517 +3179 +2979 +2356 +3745 +18262 +2186 +35356 +3436 +2152 +2123 +1452 +4729 +3761 +3136 +28010 +9340 +9339 +8710 +30400 +6267 +6269 +6268 +3757 +4755 +4754 +4026 +5117 +9277 +2947 +3386 +2217 +37483 +16002 +5687 +2072 +1909 +9122 +9123 +4131 +3912 +3229 +1880 +5688 +4332 +10800 +4985 +3108 +3475 +6080 +4790 +23053 +6081 +8190 +7017 +7283 +4730 +2159 +3429 +2660 +14145 +3484 +3762 +3222 +8322 +1421 +1859 +31765 +2914 +3051 +38201 +8881 +4340 +8074 +2678 +2677 +4110 +2731 +286 +3402 +3272 +1514 +3382 +1904 +1902 +3648 +2975 +574 +8502 +3488 +9217 +4130 +7726 +5556 +7244 +4319 +41111 +4411 +4084 +2242 +4396 +4901 +7545 +7544 +27008 +27006 +27004 +5579 +2884 +3035 +1193 +5618 +7018 +2673 +4086 +8043 +8044 +3192 +3729 +1855 +1856 +1784 +24922 +1887 +7164 +4349 +7394 +16021 +16020 +6715 +4915 +4122 +3216 +14250 +3152 +1776 +36524 +4320 +4727 +3225 +2819 +4038 +6417 +347 +3047 +2495 +10081 +38202 +19790 +2515 +2514 +4353 +38472 +10102 +4085 +3953 +4788 +3088 +3134 +3639 +4309 +2755 +1928 +5075 +26486 +5401 +3759 +43440 +1926 +1982 +1798 +9981 +4536 +4535 +1504 +592 +1267 +6935 +2036 +6316 +2221 +44818 +34980 +2380 +2379 +6107 +1772 +8416 +8417 +8266 +4023 +3629 +9617 +3679 +3727 +4942 +4941 +4940 +43439 +3628 +3620 +5116 +3259 +4666 +4669 +3819 +37601 +5084 +5085 +3383 +5599 +5600 +5601 +3665 +1818 +3044 +1295 +7962 +7117 +121 +17754 +6636 +6635 +20480 +23333 +3585 +6322 +6321 +4091 +4092 +140 +6656 +3693 +11623 +11723 +13218 +3682 +3218 +9083 +3197 +3198 +394 +2526 +7700 +7707 +2916 +2917 +4370 +6515 +12010 +5398 +3564 +4346 +1378 +1893 +3525 +3638 +2228 +6632 +3392 +3671 +6159 +3462 +3461 +3464 +3465 +3460 +3463 +3123 +34567 +8149 +6703 +6702 +2263 +3477 +3524 +6160 +17729 +3711 +45678 +2168 +3328 +38462 +3932 +3295 +2164 +3395 +2874 +3246 +3247 +4191 +4028 +3489 +4556 +5684 +13929 +31685 +9987 +4060 +13819 +13820 +13821 +13818 +13822 +2420 +7547 +3685 +2193 +4427 +1930 +8913 +7021 +7020 +5719 +5565 +5245 +6326 +6320 +6325 +3522 +44544 +13400 +6088 +3568 +8567 +3567 +5567 +7165 +4142 +3161 +5352 +195 +1172 +5993 +3199 +3574 +4059 +1177 +3624 +19999 +4646 +21212 +246 +5107 +14002 +7171 +3448 +3336 +3335 +3337 +198 +197 +3447 +5031 +4605 +2464 +2227 +3223 +1335 +2226 +33333 +2762 +2761 +3227 +3228 +33331 +2861 +2860 +2098 +4301 +3252 +547 +546 +6785 +8750 +4330 +3776 +24850 +8805 +2763 +4167 +2092 +3444 +8415 +3714 +1278 +5700 +3668 +7569 +365 +8894 +8893 +8891 +8890 +11202 +3988 +1160 +3938 +6117 +6624 +6625 +2073 +461 +3612 +3578 +11109 +2229 +1775 +2764 +3678 +6511 +1133 +29999 +2594 +3881 +3498 +8732 +2378 +3394 +3393 +2298 +2297 +9388 +9387 +3120 +3297 +1898 +8442 +9888 +4183 +4673 +3778 +5271 +3127 +1932 +4451 +2563 +4452 +9346 +7022 +3631 +3630 +105 +3271 +2699 +3004 +2129 +4187 +1724 +3113 +2314 +8380 +8377 +8376 +8379 +8378 +20810 +3818 +41797 +41796 +38002 +3364 +3366 +2824 +2823 +3609 +4055 +4054 +4053 +2654 +19220 +9093 +3183 +2565 +4078 +4774 +2153 +17222 +7551 +7563 +3072 +4047 +9695 +4846 +5992 +5683 +4692 +3191 +3417 +7169 +3973 +46998 +16384 +3947 +47100 +6970 +2491 +7023 +10321 +42508 +3822 +2417 +2555 +3257 +3256 +22343 +64 +7215 +20003 +4450 +3751 +3605 +2534 +3490 +4419 +7689 +21213 +7574 +3377 +3779 +44444 +3039 +2415 +2183 +26257 +3576 +3575 +2976 +7168 +8501 +164 +3384 +7550 +45514 +356 +2617 +3730 +6688 +6687 +6690 +7683 +2052 +3481 +4136 +4137 +9087 +172 +1729 +4980 +7229 +7228 +24754 +2897 +7279 +2512 +2513 +4870 +22305 +5787 +6633 +131 +15555 +4051 +4785 +43441 +5784 +7546 +8017 +3887 +5194 +1743 +2891 +3770 +1377 +4316 +4314 +3099 +1572 +39063 +1891 +1892 +3349 +18241 +18243 +18242 +18185 +5505 +6556 +562 +531 +3772 +5065 +5064 +2182 +3893 +2921 +2922 +13832 +4074 +4140 +4115 +3056 +3616 +3559 +4970 +4969 +3114 +3750 +12168 +2122 +7129 +7162 +7167 +5270 +1197 +9060 +3106 +12546 +5247 +5246 +3290 +4728 +8998 +8610 +8609 +3756 +8614 +8613 +8612 +8611 +1872 +3583 +24676 +4377 +5079 +4378 +1734 +3545 +7262 +3675 +2552 +22537 +3709 +14414 +5251 +1882 +42509 +2318 +4326 +1563 +7163 +1554 +7161 +595 +348 +282 +8026 +5249 +5248 +5154 +10880 +3626 +4990 +3107 +6410 +6409 +6408 +6407 +6406 +6405 +6404 +4677 +581 +4671 +2964 +2965 +28589 +47808 +3966 +2446 +1854 +1961 +2444 +2277 +4175 +3188 +3043 +9380 +3692 +5682 +2155 +4104 +4103 +4102 +3593 +2845 +2844 +4186 +2218 +4678 +2017 +2913 +7648 +4914 +7687 +6501 +9750 +3344 +1896 +4568 +10128 +6768 +6767 +3182 +1313 +3181 +2059 +3604 +6300 +10129 +3695 +6301 +2494 +2625 +48129 +8195 +2369 +2574 +5750 +13823 +13216 +4027 +5068 +25955 +25954 +6946 +3411 +24577 +5429 +2259 +4621 +6784 +4676 +4675 +4784 +3785 +5425 +5424 +4305 +3960 +3408 +5584 +5585 +1943 +3124 +6508 +6507 +4155 +1120 +1929 +4324 +10439 +6506 +6505 +6122 +4971 +3387 +152 +2635 +2169 +6696 +2204 +3512 +2071 +10260 +35100 +4195 +3277 +3502 +2066 +2238 +4413 +20057 +2992 +2050 +3965 +10990 +31020 +4685 +1140 +7508 +16003 +4071 +3104 +3437 +5067 +33123 +1146 +44600 +2264 +7543 +2419 +32896 +2317 +3821 +4937 +1520 +11367 +4154 +3617 +20999 +1170 +1171 +2864 +27876 +4485 +4704 +7235 +3087 +45000 +4405 +4404 +4406 +4402 +4403 +4400 +5727 +11489 +2192 +4077 +4448 +3581 +5150 +13702 +3451 +386 +8211 +7166 +3518 +27782 +3176 +9292 +3174 +9295 +9294 +3426 +8423 +3140 +7570 +421 +2114 +6344 +2581 +2582 +11321 +384 +23546 +1834 +1115 +4165 +1557 +3758 +7847 +5086 +4849 +2037 +1447 +3312 +187 +4488 +2336 +387 +208 +207 +203 +3454 +10548 +4674 +38203 +3239 +3236 +3237 +3238 +4573 +2758 +10252 +2759 +8121 +2754 +8122 +3184 +42999 +539 +6082 +18888 +9952 +9951 +7846 +7845 +6549 +5456 +5455 +5454 +4851 +5913 +5072 +3939 +2247 +1206 +3715 +2646 +3054 +5671 +8040 +376 +2640 +30004 +30003 +5192 +4393 +4392 +4391 +4394 +1931 +5506 +8301 +4563 +35355 +4011 +7799 +3265 +9209 +693 +36001 +9956 +9955 +6627 +3234 +2667 +2668 +3613 +4804 +2887 +3416 +3833 +9216 +2846 +17555 +2786 +3316 +3021 +3026 +4878 +3917 +4362 +7775 +3224 +23457 +23456 +4549 +4431 +2295 +3573 +5073 +3760 +3357 +3954 +3705 +3704 +2692 +6769 +33890 +7170 +2521 +2085 +3096 +2810 +2859 +3431 +9389 +3655 +5106 +5103 +44445 +7509 +6801 +4013 +2476 +2475 +2334 +12007 +12008 +6868 +4046 +18463 +32483 +4030 +8793 +62 +1955 +3781 +3619 +3618 +28119 +4726 +4502 +4597 +4598 +3598 +3597 +3125 +4149 +9953 +23294 +2933 +2934 +5783 +5782 +5785 +5781 +15363 +48049 +2339 +5265 +5264 +1181 +3446 +3428 +15998 +3091 +2133 +3774 +317 +3832 +508 +3721 +1619 +1716 +2279 +3412 +2327 +6558 +2130 +1760 +5413 +2396 +2923 +3378 +3466 +2504 +2720 +4871 +7395 +3926 +1727 +1326 +2518 +1890 +2781 +565 +4984 +3342 +21845 +1963 +2851 +3748 +1739 +1269 +2455 +2547 +2548 +2546 +7779 +2695 +312 +2996 +2893 +1589 +2649 +1224 +1345 +3625 +2538 +3321 +175 +1868 +4344 +1853 +3058 +3802 +78 +2770 +3270 +575 +1771 +4839 +4838 +4837 +671 +430 +431 +2745 +2648 +3356 +1957 +2820 +1978 +2927 +2499 +2437 +2138 +2110 +1797 +1737 +483 +390 +1867 +1624 +1833 +2879 +2767 +2768 +2943 +1568 +2489 +1237 +2741 +2742 +8804 +1588 +6069 +1869 +2642 +20670 +594 +2885 +2669 +476 +2798 +3083 +3082 +3081 +2361 +5104 +1758 +7491 +1728 +5428 +1946 +559 +1610 +3144 +1922 +2726 +6149 +1838 +4014 +1274 +2647 +4106 +6102 +4548 +19540 +1866 +6965 +6966 +6964 +6963 +1751 +1625 +5453 +2709 +7967 +3354 +566 +4178 +2986 +1226 +1836 +1654 +2838 +1692 +3644 +6071 +477 +478 +2507 +1923 +3193 +2653 +2636 +1621 +3379 +2533 +2892 +2452 +1684 +2333 +22000 +1553 +3536 +11201 +2775 +2942 +2941 +2940 +2939 +2938 +2613 +426 +4116 +4412 +1966 +3065 +1225 +1705 +1618 +1660 +2545 +2676 +3687 +2756 +1599 +2832 +2831 +2830 +2829 +5461 +2974 +498 +1626 +3595 +160 +153 +3326 +1714 +3172 +3173 +3171 +3170 +3169 +2235 +6108 +169 +5399 +2471 +558 +2308 +1681 +2385 +3562 +5024 +5025 +5427 +3391 +3744 +1646 +3275 +3698 +2390 +1793 +1647 +1697 +1693 +1695 +1696 +2919 +9599 +2423 +3844 +2959 +2818 +1817 +521 +3147 +3163 +2886 +283 +2837 +2543 +2928 +2240 +1343 +2321 +3467 +9753 +1530 +2872 +1595 +2900 +1341 +2935 +3059 +2724 +3385 +2765 +368 +2461 +2462 +1253 +2680 +3009 +2434 +2694 +2351 +2353 +2354 +1788 +2352 +3662 +2355 +2091 +1732 +8183 +1678 +2588 +2924 +2687 +5071 +1777 +2899 +494 +3875 +2937 +5437 +5436 +3469 +3285 +1293 +5272 +2865 +321 +1280 +1779 +6432 +1230 +2843 +3033 +2566 +1562 +3085 +3892 +1246 +1564 +8160 +1633 +9997 +9996 +7511 +5236 +3955 +2956 +2954 +2953 +5310 +2951 +2936 +6951 +2413 +2407 +1597 +1570 +2398 +1809 +1575 +1754 +1748 +22001 +3855 +2368 +8764 +6653 +5314 +2267 +3244 +2661 +2364 +506 +2322 +2498 +3305 +183 +650 +2329 +5991 +1463 +159 +8450 +1917 +1921 +2839 +2503 +25903 +25901 +25902 +2556 +2672 +1690 +2360 +2671 +1669 +1665 +1286 +4138 +2592 +61441 +61439 +61440 +2983 +5465 +1843 +1842 +1841 +2061 +1329 +2451 +3701 +3066 +2442 +5771 +2450 +489 +8834 +1285 +3262 +2881 +2883 +43189 +6064 +1591 +1744 +405 +2397 +2683 +2162 +1288 +2286 +2236 +167 +1685 +1831 +2981 +467 +1574 +2743 +19398 +2469 +2460 +1477 +1478 +5720 +3535 +1582 +1731 +679 +2684 +2686 +2681 +2685 +1952 +9397 +9344 +2952 +2579 +2561 +1235 +367 +8665 +471 +2926 +1815 +7786 +8033 +1581 +7979 +1534 +490 +3070 +349 +1824 +2511 +1897 +6070 +2118 +2117 +1231 +24003 +24004 +24006 +24000 +3594 +24002 +24001 +24005 +5418 +2698 +8763 +1820 +1899 +2587 +8911 +8910 +1593 +2535 +4181 +3565 +2559 +3069 +2620 +1298 +2540 +2541 +2125 +1487 +2283 +2284 +2285 +2281 +2282 +2813 +5355 +2814 +2795 +1555 +1968 +2611 +245 +4042 +1682 +1485 +2560 +2841 +2370 +2842 +2840 +398 +2424 +1773 +1649 +287 +2656 +2213 +2822 +1289 +3471 +3470 +3042 +4114 +6962 +6961 +1567 +2808 +1706 +2406 +2508 +2506 +1623 +13160 +2166 +2866 +2982 +1275 +1573 +4348 +1828 +3084 +1609 +2853 +3589 +147 +3501 +1643 +1642 +1245 +43190 +2962 +2963 +576 +2549 +1579 +1585 +503 +1907 +3202 +3548 +3060 +2652 +2633 +16991 +495 +1602 +1490 +2793 +18881 +2854 +2319 +2233 +3345 +2454 +8130 +8131 +2127 +2970 +2932 +3164 +1710 +11319 +27345 +2801 +1284 +2995 +3797 +2966 +2590 +549 +1725 +2337 +3130 +5813 +25008 +25007 +25006 +25005 +25004 +25003 +25002 +25009 +6850 +1344 +1604 +8733 +2572 +1260 +1586 +1726 +6999 +6998 +2140 +2139 +2141 +1577 +4180 +4827 +1877 +2715 +19412 +19410 +19411 +5404 +5403 +2985 +1803 +2744 +6790 +2575 +12172 +1789 +35000 +1281 +14937 +14936 +263 +375 +5094 +1816 +2245 +1238 +2778 +9321 +2643 +2421 +488 +1850 +2458 +41 +2519 +6109 +1774 +2833 +3862 +3381 +1590 +2626 +1738 +2732 +19539 +2849 +2358 +1786 +1787 +1657 +2429 +1747 +1746 +5408 +5407 +2359 +24677 +1874 +2946 +2509 +1873 +2747 +2751 +2750 +2748 +2749 +9396 +3067 +1848 +9374 +2510 +2615 +1689 +4682 +3350 +24242 +3401 +3294 +3293 +5503 +5504 +5746 +5745 +2344 +7437 +3353 +2689 +3873 +1561 +1915 +2792 +10103 +26260 +26261 +589 +1948 +2666 +26489 +26487 +2769 +2674 +6066 +1876 +2835 +2834 +2782 +16309 +2969 +2867 +2797 +2950 +1822 +1342 +5135 +2650 +2109 +2051 +2912 +309 +1865 +3289 +1804 +3286 +1740 +2211 +2707 +1273 +2181 +2553 +2896 +2858 +3610 +2651 +1325 +2445 +1265 +3053 +1292 +1878 +4098 +1780 +1795 +4099 +1821 +2151 +1227 +436 +2287 +32636 +1489 +1263 +5419 +3041 +2496 +3287 +6073 +2234 +242 +1844 +2362 +11112 +1941 +3046 +1945 +6072 +2960 +5426 +2753 +3298 +1702 +1256 +1254 +1266 +2562 +1656 +1655 +579 +1255 +1415 +2365 +2345 +6104 +8132 +1908 +3282 +1857 +1679 +2870 +3458 +5420 +772 +3645 +551 +1686 +3773 +4379 +1851 +3022 +2807 +2890 +1837 +2955 +3145 +1471 +1468 +40841 +40842 +40843 +2422 +6253 +455 +2746 +3201 +5984 +2324 +3288 +5412 +2137 +1648 +1802 +4308 +48556 +2757 +1757 +1294 +7174 +1944 +371 +504 +1741 +2931 +3020 +17219 +3903 +1768 +1767 +1766 +1765 +2856 +1640 +1639 +1794 +3987 +2571 +2412 +3315 +2116 +3061 +2836 +3450 +3105 +1756 +9283 +2906 +588 +1202 +1375 +2803 +2536 +1252 +2619 +1323 +2990 +1304 +2961 +6402 +6403 +3561 +1770 +1769 +2877 +10288 +2911 +2032 +2663 +2662 +1962 +310 +357 +354 +482 +2414 +2852 +1951 +1704 +3327 +573 +567 +2708 +2131 +2772 +3643 +1749 +5042 +1913 +2624 +1826 +2136 +2616 +9164 +9163 +9162 +1781 +2929 +1320 +2848 +2268 +459 +1536 +2639 +6831 +10080 +1845 +1653 +1849 +463 +2740 +2473 +2783 +1481 +2785 +2331 +7107 +1219 +3279 +5411 +2796 +2149 +7781 +1205 +4108 +4885 +1546 +2894 +1601 +2878 +5605 +5604 +5602 +5603 +3284 +1742 From 66d125f61795fb0c7761e5abfdc77ce73805d8f8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 2 Jul 2024 21:16:28 -0400 Subject: [PATCH 036/238] fixing handling of redirects/location headers --- bbot/modules/internal/excavate.py | 96 ++++++++++--------- .../module_tests/test_module_excavate.py | 3 - 2 files changed, 52 insertions(+), 47 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 9d4c48f3c..04a01dc60 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -756,38 +756,17 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon async def handle_event(self, event): data = event.data - # handle redirects - web_spider_distance = getattr(event, "web_spider_distance", 0) - num_redirects = max(getattr(event, "num_redirects", 0), web_spider_distance) - location = getattr(event, "redirect_location", "") - # if it's a redirect - if location: - # get the url scheme - scheme = self.helpers.is_uri(location, return_scheme=True) - if scheme in ("http", "https"): - if num_redirects <= self.max_redirects: - # tag redirects to out-of-scope hosts as affiliates - url_event = self.make_event(location, "URL_UNVERIFIED", event, tags="affiliate") - if url_event is not None: - # inherit web spider distance from parent (don't increment) - parent_web_spider_distance = getattr(event, "web_spider_distance", 0) - url_event.web_spider_distance = parent_web_spider_distance - await self.emit_event( - url_event, - context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', - ) - else: - self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}") # process response data body = event.data.get("body", "") headers = event.data.get("header-dict", "") - headers_str = event.data.get("raw_header", "") + if body == "" and headers == "": return self.assigned_cookies = {} content_type = None + reported_location_header = False for k, v in headers.items(): if k.lower() == "set_cookie": if "=" not in v: @@ -813,35 +792,64 @@ async def handle_event(self, event): else: self.debug(f"blocked cookie parameter [{cookie_name}] due to BL match") if k.lower() == "location": - for ( - method, - parsed_url, - parameter_name, - original_value, - regex_name, - additional_params, - ) in extract_params_location(v, event.parsed_url): - if self.in_bl(parameter_name) == False: - description = f"HTTP Extracted Parameter [{parameter_name}] (Location Header)" - data = { - "host": parsed_url.hostname, - "type": "GETPARAM", - "name": parameter_name, - "original_value": original_value, - "url": self.url_unparse("GETPARAM", parsed_url), - "description": description, - "additional_params": additional_params, - } - context = f"Excavate parsed a location header for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" - await self.emit_event(data, "WEB_PARAMETER", event, context=context) + redirect_location = getattr(event, "redirect_location", "") + if redirect_location: + scheme = self.helpers.is_uri(redirect_location, return_scheme=True) + if scheme in ("http", "https"): + web_spider_distance = getattr(event, "web_spider_distance", 0) + num_redirects = max(getattr(event, "num_redirects", 0), web_spider_distance) + if num_redirects <= self.max_redirects: + # we do not want to allow the web_spider_distance to be incremented on redirects, so we do not add spider-danger tag + url_event = self.make_event(redirect_location, "URL_UNVERIFIED", event, tags="affiliate") + if url_event is not None: + reported_location_header = True + await self.emit_event( + url_event, + context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', + ) + + # Try to extract parameters from the redirect URL + for ( + method, + parsed_url, + parameter_name, + original_value, + regex_name, + additional_params, + ) in extract_params_location(v, event.parsed_url): + if self.in_bl(parameter_name) == False: + description = f"HTTP Extracted Parameter [{parameter_name}] (Location Header)" + data = { + "host": parsed_url.hostname, + "type": "GETPARAM", + "name": parameter_name, + "original_value": original_value, + "url": self.url_unparse("GETPARAM", parsed_url), + "description": description, + "additional_params": additional_params, + } + context = f"Excavate parsed a location header for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) + + else: + self.warning("location header found but missing redirect_location in HTTP_RESPONSE") if k.lower() == "content-type": content_type = headers["content-type"] + await self.search( body, event, content_type, discovery_context="HTTP response (body)", ) + + if reported_location_header: + # Location header should be removed if we already found and emitted a result. + # Failure to do so results in a race against the same URL extracted by the URLExtractor submodule + # If the extracted URL wins, it will cause the manual one to be a dupe, but it will have a higher web_spider_distance. + headers.pop("location") + headers_str = "\n".join(f"{k}: {v}" for k, v in headers.items()) + await self.search( headers_str, event, diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 16a1810b9..1fc40317e 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -259,9 +259,6 @@ async def setup_before_prep(self, module_test): def check(self, module_test, events): url_unverified_events = [e for e in events if e.type == "URL_UNVERIFIED"] - for u in url_unverified_events: - self.log.critical(u) - # base URL + 25 links = 26 assert len(url_unverified_events) == 26 url_data = [e.data for e in url_unverified_events if "spider-max" not in e.tags and "spider-danger" in e.tags] From 048fcb56f3e5ecbd7b85a9ee07b821ba647fe6bd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 3 Jul 2024 11:06:54 -0400 Subject: [PATCH 037/238] validator docstring --- bbot/core/event/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index dde5ccb3b..bcd8daa7c 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -518,6 +518,14 @@ def parent_id(self): @property def validators(self): + """ + Depending on whether the scan attribute is accessible, return either a config-aware or non-config-aware validator + + This exists to prevent a chicken-and-egg scenario during the creation of certain events such as URLs, + whose sanitization behavior is different depending on the config. + + However, thanks to this property, validation can still work in the absence of a config. + """ if self.scan is not None: return self.scan.helpers.config_aware_validators return validators @@ -1074,14 +1082,12 @@ def add_tag(self, tag): @property def is_spider_max(self): - if self.scan: web_spider_distance = self.scan.config.get("web_spider_distance", 0) web_spider_depth = self.scan.config.get("web_spider_depth", 1) depth = url_depth(self.parsed_url) if (self.web_spider_distance > web_spider_distance) or (depth > web_spider_depth): return True - return False def with_port(self): From f5c6dc9b0b2095c1de32556b69c0deb46cedc0cf Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 10:05:25 -0400 Subject: [PATCH 038/238] detaching discovery_context from rule object --- bbot/modules/internal/excavate.py | 74 ++++++++++++++++++------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 04a01dc60..1a5605004 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -73,8 +73,6 @@ def __init__(self, excavate): self.name = "" async def preprocess(self, r, event, discovery_context): - self.discovery_context = discovery_context - description = "" tags = [] emit_match = False @@ -90,17 +88,17 @@ async def preprocess(self, r, event, discovery_context): yara_results = {} for h in r.strings: yara_results[h.identifier.lstrip("$")] = sorted(set([i.matched_data.decode("utf-8") for i in h.instances])) - await self.process(yara_results, event, yara_rule_settings) + await self.process(yara_results, event, yara_rule_settings, discovery_context) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): for result in results: event_data = {"host": str(event.host), "url": event.data.get("url", "")} - event_data["description"] = f"{self.discovery_context} {yara_rule_settings.description}" + event_data["description"] = f"{discovery_context} {yara_rule_settings.description}" if yara_rule_settings.emit_match: event_data["description"] += f" [{result}]" - await self.report(event_data, event, yara_rule_settings) + await self.report(event_data, event, yara_rule_settings, discovery_context) async def report_prep(self, event_data, event_type, event, tags): event_draft = self.excavate.make_event(event_data, event_type, parent=event) @@ -109,15 +107,17 @@ async def report_prep(self, event_data, event_type, event, tags): event_draft.tags = tags return event_draft - async def report(self, event_data, event, yara_rule_settings, event_type="FINDING", abort_if=None, **kwargs): + async def report( + self, event_data, event, yara_rule_settings, discovery_context, event_type="FINDING", abort_if=None, **kwargs + ): # If a description is not set and is needed, provide a basic one if event_type == "FINDING" and "description" not in event_data.keys(): - event_data["description"] = f"{self.discovery_context} {yara_rule_settings['self.description']}" + event_data["description"] = f"{discovery_context} {yara_rule_settings['self.description']}" subject = "" if isinstance(event_data, str): subject = f" event_data" - context = f"Excavate's [{self.__class__.__name__}] submodule emitted [{event_type}]{subject}, because {self.discovery_context} {yara_rule_settings.description}" + context = f"Excavate's [{self.__class__.__name__}] submodule emitted [{event_type}]{subject}, because {discovery_context} {yara_rule_settings.description}" tags = yara_rule_settings.tags event_draft = await self.report_prep(event_data, event_type, event, tags, **kwargs) if event_draft: @@ -129,7 +129,7 @@ class CustomExtractor(ExcavateRule): def __init__(self, excavate): super().__init__(excavate) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): for result in results: @@ -142,7 +142,7 @@ async def process(self, yara_results, event, yara_rule_settings): ) if yara_rule_settings.emit_match: event_data["description"] += f" and extracted [{result}]" - await self.report(event_data, event, yara_rule_settings) + await self.report(event_data, event, yara_rule_settings, discovery_context) class excavate(BaseInternalModule): @@ -333,7 +333,7 @@ def __init__(self, excavate): rf'rule parameter_extraction {{meta: description = "contains POST form" strings: {regexes_component} condition: any of them}}' ) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): for result in results: if identifier not in self.parameterExtractorCallbackDict.keys(): @@ -376,7 +376,9 @@ async def process(self, yara_results, event, yara_rule_settings): "assigned_cookies": self.excavate.assigned_cookies, "description": description, } - await self.report(data, event, yara_rule_settings, event_type="WEB_PARAMETER") + await self.report( + data, event, yara_rule_settings, discovery_context, event_type="WEB_PARAMETER" + ) else: self.excavate.debug(f"blocked parameter [{parameter_name}] due to BL match") else: @@ -387,13 +389,13 @@ class CSPExtractor(ExcavateRule): "csp": r'rule csp { meta: tags = "affiliate" description = "contains CSP Header" strings: $csp = /Content-Security-Policy:[^\r\n]+/ nocase condition: $csp }', } - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier in yara_results.keys(): for csp_str in yara_results[identifier]: domains = await self.helpers.re.findall(bbot_regexes.dns_name_regex, csp_str) unique_domains = set(domains) for domain in unique_domains: - await self.report(domain, event, yara_rule_settings, event_type="DNS_NAME") + await self.report(domain, event, yara_rule_settings, discovery_context, event_type="DNS_NAME") class EmailExtractor(ExcavateRule): @@ -401,10 +403,12 @@ class EmailExtractor(ExcavateRule): "email": 'rule email { meta: description = "contains email address" strings: $email = /[^\\W_][\\w\\-\\.\\+\']{0,100}@[a-zA-Z0-9\\-]{1,100}(\\.[a-zA-Z0-9\\-]{1,100})*\\.[a-zA-Z]{2,63}/ nocase fullword condition: $email }', } - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier in yara_results.keys(): for email_str in yara_results[identifier]: - await self.report(email_str, event, yara_rule_settings, event_type="EMAIL_ADDRESS") + await self.report( + email_str, event, yara_rule_settings, discovery_context, event_type="EMAIL_ADDRESS" + ) # Future Work: Emit a JWT Object, and make a new Module to ingest it. class JWTExtractor(ExcavateRule): @@ -442,15 +446,15 @@ def __init__(self, excavate): f'rule error_detection {{meta: description = "contains a verbose error message" strings: {signature_component} condition: any of them}}' ) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier in yara_results.keys(): for findings in yara_results[identifier]: event_data = { "host": str(event.host), "url": event.data.get("url", ""), - "description": f"{self.discovery_context} {yara_rule_settings.description} ({identifier})", + "description": f"{discovery_context} {yara_rule_settings.description} ({identifier})", } - await self.report(event_data, event, yara_rule_settings, event_type="FINDING") + await self.report(event_data, event, yara_rule_settings, discovery_context, event_type="FINDING") class SerializationExtractor(ExcavateRule): @@ -474,15 +478,15 @@ def __init__(self, excavate): f'rule serialization_detection {{meta: description = "contains a possible serialized object" strings: {regexes_component} condition: any of them}}' ) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier in yara_results.keys(): for findings in yara_results[identifier]: event_data = { "host": str(event.host), "url": event.data.get("url", ""), - "description": f"{self.discovery_context} {yara_rule_settings.description} ({identifier})", + "description": f"{discovery_context} {yara_rule_settings.description} ({identifier})", } - await self.report(event_data, event, yara_rule_settings, event_type="FINDING") + await self.report(event_data, event, yara_rule_settings, discovery_context, event_type="FINDING") class FunctionalityExtractor(ExcavateRule): @@ -498,7 +502,7 @@ class NonHttpSchemeExtractor(ExcavateRule): scheme_blacklist = ["javascript", "mailto", "tel", "data", "vbscript", "about", "file"] - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): for url_str in results: scheme = url_str.split("://")[0] @@ -523,12 +527,17 @@ async def process(self, yara_results, event, yara_rule_settings): continue abort_if = lambda e: e.scope_distance > 0 finding_data = {"host": str(host), "description": f"Non-HTTP URI: {parsed_url.geturl()}"} - await self.report(finding_data, event, yara_rule_settings, abort_if=abort_if) + await self.report(finding_data, event, yara_rule_settings, discovery_context, abort_if=abort_if) protocol_data = {"protocol": parsed_url.scheme, "host": str(host)} if port: protocol_data["port"] = port await self.report( - protocol_data, event, yara_rule_settings, event_type="PROTOCOL", abort_if=abort_if + protocol_data, + event, + yara_rule_settings, + discovery_context, + event_type="PROTOCOL", + abort_if=abort_if, ) class URLExtractor(ExcavateRule): @@ -544,7 +553,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.web_spider_links_per_page = self.excavate.scan.config.get("web_spider_links_per_page", 20) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): urls_found = 0 @@ -581,7 +590,12 @@ async def process(self, yara_results, event, yara_rule_settings): urls_found += 1 await self.report( - final_url, event, yara_rule_settings, event_type="URL_UNVERIFIED", urls_found=urls_found + final_url, + event, + yara_rule_settings, + discovery_context, + event_type="URL_UNVERIFIED", + urls_found=urls_found, ) async def report_prep(self, event_data, event_type, event, tags, **kwargs): @@ -611,10 +625,10 @@ def __init__(self, excavate): f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}' ) - async def process(self, yara_results, event, yara_rule_settings): + async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier in yara_results.keys(): for domain_str in yara_results[identifier]: - await self.report(domain_str, event, yara_rule_settings, event_type="DNS_NAME") + await self.report(domain_str, event, yara_rule_settings, discovery_context, event_type="DNS_NAME") def add_yara_rule(self, rule_name, rule_content, rule_instance): rule_instance.name = rule_name From 1aab3dc9c9fc9cbbf25918de5072f471472a7654 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 10:10:59 -0400 Subject: [PATCH 039/238] removing debugging --- bbot/test/test_step_1/test_manager_scope_accuracy.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index d1483f6fa..5250c7637 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -340,9 +340,6 @@ def custom_setup(scan): assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1]) assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"]) - - for e in all_events: - log.critical(e) assert len(all_events) == 14 assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0]) assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0]) From 8ea1910f3378d7ce05c248c10d6e6f6ad8a104d9 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 3 Jul 2024 16:52:29 -0400 Subject: [PATCH 040/238] rework event dudupe initial --- bbot/core/event/base.py | 44 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index bcd8daa7c..87defd87f 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -9,12 +9,13 @@ import traceback from copy import copy +from pathlib import Path from typing import Optional from contextlib import suppress -from urllib.parse import urljoin from radixtarget import RadixTarget +from urllib.parse import urljoin, parse_qs from pydantic import BaseModel, field_validator -from pathlib import Path + from .helpers import * from bbot.errors import * @@ -1049,6 +1050,26 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.num_redirects = getattr(self.parent, "num_redirects", 0) + def _data_id(self): + # consider spider-danger tag when deduping + data = super()._data_id() + if "spider-danger" in self.tags: + data = "spider-danger" + data + + if self.scan is not None: + if not self.scan.config.get("url_querystring_remove", True) and self.parsed_url.query: + query_dict = parse_qs(self.parsed_url.query) + if self.scan.config.get("url_querystring_collapse", True): + # Only consider parameter names in dedup (collapse values) + cleaned_query = "|".join(sorted(query_dict.keys())) + else: + # Consider parameter names and values in dedup + cleaned_query = "&".join( + f"{key}={','.join(sorted(values))}" for key, values in sorted(query_dict.items()) + ) + data += f":{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" + return data + def sanitize_data(self, data): self.parsed_url = self.validators.validate_url_parsed(data) @@ -1156,26 +1177,10 @@ class URL_HINT(URL_UNVERIFIED): class WEB_PARAMETER(DictHostEvent): def _data_id(self): - # dedupe by url:name:param_type url = self.data.get("url", "") name = self.data.get("name", "") param_type = self.data.get("type", "") - # REMOVE - # this is a hack which needs to be replaced with a real fix in bbot-2.0 branch - if self.scan is not None: - if self.scan.config.get("url_querystring_remove", True) == False: - from urllib.parse import urlparse - - parsed_url = urlparse(url) - - if self.scan.config.get("url_querystring_collapse", True) == True: - # Only consider parameter names in dedup (collapse values) - cleaned_query = "|".join(sorted([p.split("=")[0] for p in parsed_url.query.split("&")])) - else: - # Consider parameter names and values in dedup - cleaned_query = "&".join(sorted(parsed_url.query.split("&"), key=lambda p: p.split("=")[0])) - url = f"{parsed_url.scheme}:{parsed_url.netloc}:{parsed_url.path}:{cleaned_query}" return f"{url}:{name}:{param_type}" def _url(self): @@ -1208,6 +1213,9 @@ def __init__(self, *args, **kwargs): if str(self.http_status).startswith("3"): self.num_redirects += 1 + def _data_id(self): + return self.data["url"] + def sanitize_data(self, data): url = data.get("url", "") self.parsed_url = self.validators.validate_url_parsed(url) From fb796f09468685bbf3142184b8875fc5e91306f8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 3 Jul 2024 22:09:37 -0400 Subject: [PATCH 041/238] fixing test --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 1fc40317e..5e968b944 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -640,9 +640,14 @@ def check(self, module_test, events): found_url_unverified_dummy = False found_url_event = False - assert self.dummy_module.events_seen == ["http://127.0.0.1:8888/", "http://127.0.0.1:8888/spider"] + assert sorted(self.dummy_module.events_seen) == [ + "http://127.0.0.1:8888/", + "http://127.0.0.1:8888/spider", + "http://127.0.0.1:8888/spider", + ] for e in events: + if e.type == "URL_UNVERIFIED": if e.data == "http://127.0.0.1:8888/spider": From 75e9566bf26237f786013966a04e02380c021e96 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 3 Jul 2024 22:14:36 -0400 Subject: [PATCH 042/238] spacing --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 5e968b944..537fe173d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -647,9 +647,7 @@ def check(self, module_test, events): ] for e in events: - if e.type == "URL_UNVERIFIED": - if e.data == "http://127.0.0.1:8888/spider": if str(e.module) == "excavate" and "spider-danger" in e.tags and "spider-max" in e.tags: found_url_unverified_spider_max = True From 582a55c318f44a82cc2f48db5470dd2048370ba8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 3 Jul 2024 23:17:43 -0400 Subject: [PATCH 043/238] fixing querystring_collapse dedupe, adding tests --- bbot/core/event/base.py | 3 +- .../module_tests/test_module_excavate.py | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 87defd87f..0238d5c95 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1052,7 +1052,8 @@ def __init__(self, *args, **kwargs): def _data_id(self): # consider spider-danger tag when deduping - data = super()._data_id() + data = super()._data_id().split("?")[0] + if "spider-danger" in self.tags: data = "spider-danger" + data diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 537fe173d..6d5ff258f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -222,6 +222,42 @@ def check(self, module_test, events): assert 0 == len([e for e in events if e.type == "PROTOCOL" and e.data["protocol"] == "SSH"]) +class TestExcavateQuerystringRemoveTrue(TestExcavate): + targets = ["http://127.0.0.1:8888/"] + config_overrides = {"url_querystring_remove": True} + lots_of_params = """ + + + + + + + + + + + """ + + async def setup_before_prep(self, module_test): + module_test.httpserver.expect_request("/").respond_with_data(self.lots_of_params) + + def check(self, module_test, events): + assert len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/endpoint"]) == 1 + +class TestExcavateQuerystringRemoveFalse(TestExcavateQuerystringRemoveTrue): + config_overrides = {"url_querystring_remove": False, "url_querystring_collapse": True} + + def check(self, module_test, events): + assert len([e for e in events if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?")]) == 1 + +class TestExcavateQuerystringCollapseFalse(TestExcavateQuerystringRemoveTrue): + config_overrides = {"url_querystring_remove": False, "url_querystring_collapse": False} + + def check(self, module_test, events): + assert len([e for e in events if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?")]) == 10 + + + class TestExcavateMaxLinksPerPage(TestExcavate): targets = ["http://127.0.0.1:8888/"] config_overrides = {"web_spider_links_per_page": 10, "web_spider_distance": 1} From 6d0b87d6e9f1fba0dd0ebb82f7e1c7126ed69e17 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 3 Jul 2024 23:19:52 -0400 Subject: [PATCH 044/238] black --- .../module_tests/test_module_excavate.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 6d5ff258f..d12aef7c4 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -242,20 +242,41 @@ async def setup_before_prep(self, module_test): module_test.httpserver.expect_request("/").respond_with_data(self.lots_of_params) def check(self, module_test, events): - assert len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/endpoint"]) == 1 + assert ( + len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/endpoint"]) == 1 + ) + class TestExcavateQuerystringRemoveFalse(TestExcavateQuerystringRemoveTrue): config_overrides = {"url_querystring_remove": False, "url_querystring_collapse": True} def check(self, module_test, events): - assert len([e for e in events if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?")]) == 1 + assert ( + len( + [ + e + for e in events + if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?") + ] + ) + == 1 + ) + class TestExcavateQuerystringCollapseFalse(TestExcavateQuerystringRemoveTrue): config_overrides = {"url_querystring_remove": False, "url_querystring_collapse": False} def check(self, module_test, events): - assert len([e for e in events if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?")]) == 10 - + assert ( + len( + [ + e + for e in events + if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?") + ] + ) + == 10 + ) class TestExcavateMaxLinksPerPage(TestExcavate): From 667501fa104d99b223987f715c1b0a9882f491b7 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 00:23:22 -0400 Subject: [PATCH 045/238] fixing bug with URL dedupe --- bbot/core/event/base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 0238d5c95..9c5359876 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1051,9 +1051,13 @@ def __init__(self, *args, **kwargs): self.num_redirects = getattr(self.parent, "num_redirects", 0) def _data_id(self): - # consider spider-danger tag when deduping - data = super()._data_id().split("?")[0] + data = super()._data_id() + # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings) + if self.__class__.__name__.startswith("URL"): + data == data.split("?")[0] + + # consider spider-danger tag when deduping if "spider-danger" in self.tags: data = "spider-danger" + data From b4d929ec278def897bbcc5e5169ee4ab261b0ca2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 09:45:30 -0400 Subject: [PATCH 046/238] more fixes for URL dedupe --- bbot/core/event/base.py | 14 +++++++------- .../module_tests/test_module_excavate.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 9c5359876..2f7226617 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1053,15 +1053,15 @@ def __init__(self, *args, **kwargs): def _data_id(self): data = super()._data_id() + # remove the querystring for URL/URL_UNVERIFIED events, because we will conditionally add it back in (based on settings) - if self.__class__.__name__.startswith("URL"): - data == data.split("?")[0] + if self.__class__.__name__.startswith("URL") and self.scan is not None: + prefix = data.split("?")[0] - # consider spider-danger tag when deduping - if "spider-danger" in self.tags: - data = "spider-danger" + data + # consider spider-danger tag when deduping + if "spider-danger" in self.tags: + prefix += "spider-danger" - if self.scan is not None: if not self.scan.config.get("url_querystring_remove", True) and self.parsed_url.query: query_dict = parse_qs(self.parsed_url.query) if self.scan.config.get("url_querystring_collapse", True): @@ -1072,7 +1072,7 @@ def _data_id(self): cleaned_query = "&".join( f"{key}={','.join(sorted(values))}" for key, values in sorted(query_dict.items()) ) - data += f":{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" + data = f"{prefix}:{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" return data def sanitize_data(self, data): diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index d12aef7c4..871243f47 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -224,7 +224,7 @@ def check(self, module_test, events): class TestExcavateQuerystringRemoveTrue(TestExcavate): targets = ["http://127.0.0.1:8888/"] - config_overrides = {"url_querystring_remove": True} + config_overrides = {"url_querystring_remove": True, "url_querystring_collapse": True} lots_of_params = """ From a32bea3e9d04432faa05c04815b0fc6c865931ed Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 10:18:30 -0400 Subject: [PATCH 047/238] revising test --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 871243f47..3992274c2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -699,8 +699,7 @@ def check(self, module_test, events): assert sorted(self.dummy_module.events_seen) == [ "http://127.0.0.1:8888/", - "http://127.0.0.1:8888/spider", - "http://127.0.0.1:8888/spider", + "http://127.0.0.1:8888/spider" ] for e in events: From e6a001990407c6ff10c248579f6c9f5dd6c25258 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 10:19:54 -0400 Subject: [PATCH 048/238] black --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 3992274c2..d5a6a449b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -697,10 +697,7 @@ def check(self, module_test, events): found_url_unverified_dummy = False found_url_event = False - assert sorted(self.dummy_module.events_seen) == [ - "http://127.0.0.1:8888/", - "http://127.0.0.1:8888/spider" - ] + assert sorted(self.dummy_module.events_seen) == ["http://127.0.0.1:8888/", "http://127.0.0.1:8888/spider"] for e in events: if e.type == "URL_UNVERIFIED": From 956d49191a9c1932a1ef85dfc2b5215efd612e1a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Jul 2024 11:31:35 -0400 Subject: [PATCH 049/238] small tweaks --- bbot/core/event/base.py | 2 +- bbot/test/test_step_2/module_tests/test_module_excavate.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 2f7226617..df75f9e4c 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1219,7 +1219,7 @@ def __init__(self, *args, **kwargs): self.num_redirects += 1 def _data_id(self): - return self.data["url"] + return self.data["method"] + "|" + self.data["url"] def sanitize_data(self, data): url = data.get("url", "") diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index d5a6a449b..97a0af9c2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -242,6 +242,9 @@ async def setup_before_prep(self, module_test): module_test.httpserver.expect_request("/").respond_with_data(self.lots_of_params) def check(self, module_test, events): + assert ( + len([e for e in events if e.type == "URL_UNVERIFIED"]) == 2 + ) assert ( len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/endpoint"]) == 1 ) From 7ad0dcf64d3cae319323a46087ac8d1a41f09c65 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 4 Jul 2024 11:31:47 -0400 Subject: [PATCH 050/238] blacked --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 97a0af9c2..060fafdb7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -242,9 +242,7 @@ async def setup_before_prep(self, module_test): module_test.httpserver.expect_request("/").respond_with_data(self.lots_of_params) def check(self, module_test, events): - assert ( - len([e for e in events if e.type == "URL_UNVERIFIED"]) == 2 - ) + assert len([e for e in events if e.type == "URL_UNVERIFIED"]) == 2 assert ( len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/endpoint"]) == 1 ) From 894915bb5b9f01d74b0a5a669024d63e096110aa Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 24 Jun 2024 13:13:01 -0400 Subject: [PATCH 051/238] remove parse_list_string --- bbot/core/helpers/misc.py | 52 +++++++++------------------ bbot/modules/deadly/ffuf.py | 4 +-- bbot/modules/ffuf_shortnames.py | 4 +-- bbot/test/test_step_1/test_helpers.py | 19 +++++++--- 4 files changed, 35 insertions(+), 44 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index a9bfc1ef3..05831421c 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1102,7 +1102,14 @@ def str_or_file(s): split_regex = re.compile(r"[\s,]") -def chain_lists(l, try_files=False, msg=None, remove_blank=True): +def chain_lists( + l, + try_files=False, + msg=None, + remove_blank=True, + validate=False, + validate_chars='<>:"/\\|?*)', +): """Chains together list elements, allowing for entries separated by commas. This function takes a list `l` and flattens it by splitting its entries on commas. @@ -1115,10 +1122,15 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True): try_files (bool, optional): Whether to try to open entries as files. Defaults to False. msg (str, optional): An optional message to log when reading from a file. Defaults to None. remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True. + validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False. + validate (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)' Returns: list: The list of chained elements. + Raises: + ValueError: If the input string contains invalid characters, when enabled (off by default). + Examples: >>> chain_lists(["a", "b,c,d"]) ['a', 'b', 'c', 'd'] @@ -1130,8 +1142,11 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True): l = [l] final_list = dict() for entry in l: - for s in split_regex.split(entry): + for s in entry.split(","): f = s.strip() + if validate: + if any((c in validate_chars) or (ord(c) < 32 and c != " ") for c in f): + raise ValueError(f"Invalid character in string: {f}") f_path = Path(f).resolve() if try_files and f_path.is_file(): if msg is not None: @@ -2557,39 +2572,6 @@ def parse_port_string(port_string): return ports -def parse_list_string(list_string): - """ - Parses a comma-separated string into a list, removing invalid characters. - - Args: - list_string (str): The string containing elements separated by commas. - - Returns: - list: A list of individual elements parsed from the input string. - - Raises: - ValueError: If the input string contains invalid characters. - - Examples: - >>> parse_list_string("html,js,css") - ['html', 'js', 'css'] - - >>> parse_list_string("png,jpg,gif") - ['png', 'jpg', 'gif'] - - >>> parse_list_string("invalid<>char") - ValueError: Invalid character in string: invalid<>char - """ - elements = list_string.split(",") - result = [] - - for element in elements: - if any((c in '<>:"/\\|?*') or (ord(c) < 32 and c != " ") for c in element): - raise ValueError(f"Invalid character in string: {element}") - result.append(element) - return result - - async def as_completed(coros): """ Async generator that yields completed Tasks as they are completed. diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index ece21076a..fbc75d413 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -1,5 +1,5 @@ from bbot.modules.base import BaseModule -from bbot.core.helpers.misc import parse_list_string +from bbot.core.helpers.misc import chain_lists import random import string @@ -44,7 +44,7 @@ async def setup(self): self.tempfile, tempfile_len = self.generate_templist() self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]") try: - self.extensions = parse_list_string(self.config.get("extensions", "")) + self.extensions = chain_lists(self.config.get("extensions", ""), validate=True) self.debug(f"Using custom extensions: [{','.join(self.extensions)}]") except ValueError as e: self.warning(f"Error parsing extensions: {e}") diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index 6fcb90f0f..bcee48ad4 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -3,7 +3,7 @@ import string from bbot.modules.deadly.ffuf import ffuf -from bbot.core.helpers.misc import parse_list_string +from bbot.core.helpers.misc import chain_lists def find_common_prefixes(strings, minimum_set_length=4): @@ -83,7 +83,7 @@ async def setup(self): self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions) try: - self.extensions = parse_list_string(self.config.get("extensions", "")) + self.extensions = chain_lists(self.config.get("extensions", ""), validate=True) self.debug(f"Using custom extensions: [{','.join(self.extensions)}]") except ValueError as e: return False, f"Error parsing extensions: {e}" diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index a08a4c270..a289f2ff2 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -784,17 +784,26 @@ def test_portparse(helpers): assert str(e.value) == "Invalid port or port range: foo" -def test_liststring(helpers): - assert helpers.parse_list_string("hello,world,bbot") == ["hello", "world", "bbot"] +# test chain_lists helper + +def test_liststring_valid_strings(helpers): + assert helpers.chain_lists("hello,world,bbot") == ["hello", "world", "bbot"] + + +def test_liststring_invalid_string(helpers): with pytest.raises(ValueError) as e: - helpers.parse_list_string("hello,world,\x01") + helpers.chain_lists("hello,world,\x01", validate=True) assert str(e.value) == "Invalid character in string: \x01" - assert helpers.parse_list_string("hello") == ["hello"] +def test_liststring_singleitem(helpers): + assert helpers.chain_lists("hello") == ["hello"] + + +def test_liststring_invalidfnchars(helpers): with pytest.raises(ValueError) as e: - helpers.parse_list_string("hello,world,bbot|test") + helpers.chain_lists("hello,world,bbot|test", validate=True) assert str(e.value) == "Invalid character in string: bbot|test" From 354a3c8d9275d62857ecee2d47c0dce0f7d65372 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 24 Jun 2024 13:17:01 -0400 Subject: [PATCH 052/238] replacing line --- bbot/core/helpers/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 05831421c..67e225eee 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1142,7 +1142,7 @@ def chain_lists( l = [l] final_list = dict() for entry in l: - for s in entry.split(","): + for s in split_regex.split(entry): f = s.strip() if validate: if any((c in validate_chars) or (ord(c) < 32 and c != " ") for c in f): From 9993086d2f1ddddfb76a949e14b7970fee576fad Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 24 Jun 2024 14:15:47 -0400 Subject: [PATCH 053/238] removing unnecessary imports --- bbot/modules/deadly/ffuf.py | 3 +-- bbot/modules/ffuf_shortnames.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index fbc75d413..1f23ccb7e 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -1,5 +1,4 @@ from bbot.modules.base import BaseModule -from bbot.core.helpers.misc import chain_lists import random import string @@ -44,7 +43,7 @@ async def setup(self): self.tempfile, tempfile_len = self.generate_templist() self.verbose(f"Generated dynamic wordlist with length [{str(tempfile_len)}]") try: - self.extensions = chain_lists(self.config.get("extensions", ""), validate=True) + self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True) self.debug(f"Using custom extensions: [{','.join(self.extensions)}]") except ValueError as e: self.warning(f"Error parsing extensions: {e}") diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index bcee48ad4..76e36de03 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -3,7 +3,6 @@ import string from bbot.modules.deadly.ffuf import ffuf -from bbot.core.helpers.misc import chain_lists def find_common_prefixes(strings, minimum_set_length=4): @@ -83,7 +82,7 @@ async def setup(self): self.wordlist_extensions = await self.helpers.wordlist(wordlist_extensions) try: - self.extensions = chain_lists(self.config.get("extensions", ""), validate=True) + self.extensions = self.helpers.chain_lists(self.config.get("extensions", ""), validate=True) self.debug(f"Using custom extensions: [{','.join(self.extensions)}]") except ValueError as e: return False, f"Error parsing extensions: {e}" From 297a6c0195060187162ef85cfd1fd90ea528d134 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 24 Jun 2024 14:16:47 -0400 Subject: [PATCH 054/238] doc string typo --- bbot/core/helpers/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 67e225eee..eecab89ca 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -1123,7 +1123,7 @@ def chain_lists( msg (str, optional): An optional message to log when reading from a file. Defaults to None. remove_blank (bool, optional): Whether to remove blank entries from the list. Defaults to True. validate (bool, optional): Whether to perform validation for undesirable characters. Defaults to False. - validate (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)' + validate_chars (str, optional): When performing validation, what additional set of characters to block (blocks non-printable ascii automatically). Defaults to '<>:"/\\|?*)' Returns: list: The list of chained elements. From 63368e38d236be1e209472a11ebf4047da327c56 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 27 Jun 2024 06:28:44 -0400 Subject: [PATCH 055/238] speed up fingerprintx --- bbot/modules/fingerprintx.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index cac87c5c3..bc4246211 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -18,6 +18,9 @@ class fingerprintx(BaseModule): _max_event_handlers = 2 _priority = 2 + options = {"skip_common_web": True} + options_desc = {"skip_common_web": "Skip common web ports such as 80, 443, 8080, 8443, etc."} + deps_ansible = [ { "name": "Download fingerprintx", @@ -30,6 +33,35 @@ class fingerprintx(BaseModule): }, ] + common_web_ports = ( + 80, + 443, + # cloudflare HTTP + 8080, + 8880, + 2052, + 2082, + 2086, + 2095, + # cloudflare HTTPS + 2053, + 2083, + 2087, + 2096, + 8443, + ) + + async def setup(self): + self.skip_common_web = self.config.get("skip_common_web", True) + return True + + async def filter_event(self, event): + if self.skip_common_web: + port_str = str(event.port) + if event.port in self.common_web_ports or any(port_str.endswith(x) for x in ("080", "443")): + return False, "port is a common web port and skip_common_web=True" + return True + async def handle_batch(self, *events): _input = {e.data: e for e in events} command = ["fingerprintx", "--json"] From 67f373e4f3ab181eeb197b559dd254ec0771015f Mon Sep 17 00:00:00 2001 From: BBOT Docs Autopublish Date: Thu, 27 Jun 2024 14:00:02 +0000 Subject: [PATCH 056/238] Refresh module docs --- docs/scanning/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index ba8eeb525..7a7225561 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -253,6 +253,7 @@ Many modules accept their own configuration options. These options have the abil | modules.filedownload.base_64_encoded_file | str | Stream the bytes of a file and encode them in base 64 for event data. | false | | modules.filedownload.extensions | list | File extensions to download | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'exe', 'ica', 'indd', 'ini', 'jar', 'key', 'pub', 'log', 'markdown', 'md', 'msi', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'raw', 'rdp', 'sh', 'sql', 'swp', 'sxw', 'tar', 'tar.gz', 'zip', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | | modules.filedownload.max_filesize | str | Cancel download if filesize is greater than this size | 10MB | +| modules.fingerprintx.skip_common_web | bool | Skip common web ports such as 80, 443, 8080, 8443, etc. | True | | modules.fingerprintx.version | str | fingerprintx version | 1.1.4 | | modules.gitlab.api_key | str | Gitlab access token | | | modules.gowitness.idle_timeout | int | Skip the current gowitness batch if it stalls for longer than this many seconds | 1800 | From 1a2f12165382a9fe9570e27bbd4d186308e35faa Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 2 Jul 2024 19:02:23 -0400 Subject: [PATCH 057/238] merge killing multiple modules --- bbot/cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index f3788f61a..e34d33943 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -6,6 +6,7 @@ from bbot.errors import * from bbot import __version__ from bbot.logger import log_to_stderr +from bbot.core.helpers.misc import chain_lists if multiprocessing.current_process().name == "MainProcess": @@ -191,17 +192,20 @@ async def _main(): from bbot.core.helpers.misc import smart_decode def handle_keyboard_input(keyboard_input): - kill_regex = re.compile(r"kill (?P[a-z0-9_]+)") + kill_regex = re.compile(r"kill (?P[a-z0-9_ ,]+)") if keyboard_input: log.verbose(f'Got keyboard input: "{keyboard_input}"') kill_match = kill_regex.match(keyboard_input) if kill_match: - module = kill_match.group("module") - if module in scan.modules: - log.hugewarning(f'Killing module: "{module}"') - scan.kill_module(module, message="killed by user") - else: - log.warning(f'Invalid module: "{module}"') + modules = kill_match.group("modules") + if modules: + modules = chain_lists(modules) + for module in modules: + if module in scan.modules: + log.hugewarning(f'Killing module: "{module}"') + scan.kill_module(module, message="killed by user") + else: + log.warning(f'Invalid module: "{module}"') else: scan.preset.core.logger.toggle_log_level(logger=log) scan.modules_status(_log=True) From b0d2f7380ad772f0b5f205137e7a6fb3cb2f1926 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 20:39:57 -0400 Subject: [PATCH 058/238] missing word --- docs/modules/internal_modules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/internal_modules.md b/docs/modules/internal_modules.md index d9372fac4..1a22ebb1e 100644 --- a/docs/modules/internal_modules.md +++ b/docs/modules/internal_modules.md @@ -2,7 +2,7 @@ ## What are internal modules? -Internal modules just like regular modules, except that they run all the time. They do not have to be explicitly enabled. They can, however, be explicitly disabled if needed. +Internal modules are just like regular modules, except that they run all the time. They do not have to be explicitly enabled. They can, however, be explicitly disabled if needed. Turning them off is simple, a root-level config option is present which can be set to False to disable them: From 4818b18279b3bb68fe5ff4319775c0cb25a352bd Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 20:57:03 -0400 Subject: [PATCH 059/238] removing excavate on RAW_TEXT (raw now) --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 1a5605004..2a4fcbeda 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -156,7 +156,7 @@ class excavateTestRule(ExcavateRule): } """ - watched_events = ["HTTP_RESPONSE", "RAW_TEXT"] + watched_events = ["HTTP_RESPONSE"] produced_events = ["URL_UNVERIFIED", "WEB_PARAMETER"] flags = ["passive"] meta = { From 01d7650bec6c244cbeff3c11a22b5fcc0c717270 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 21:20:26 -0400 Subject: [PATCH 060/238] removing remaining parse_list_string call --- bbot/modules/internal/excavate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 2a4fcbeda..f41489e95 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -8,8 +8,6 @@ import bbot.core.helpers.regexes as bbot_regexes from bbot.modules.internal.base import BaseInternalModule from urllib.parse import urlparse, urljoin, parse_qs, urlunparse -from bbot.core.helpers.misc import parse_list_string - def find_subclasses(obj, base_class): subclasses = [] @@ -80,7 +78,7 @@ async def preprocess(self, r, event, discovery_context): if "description" in r.meta.keys(): description = r.meta["description"] if "tags" in r.meta.keys(): - tags = parse_list_string(r.meta["tags"]) + tags = self.excavate.helpers.chain_lists(r.meta["tags"]) if "emit_match" in r.meta.keys(): emit_match = True From 8467b6f82d115d58183799be523f440fb41aede1 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 21:23:25 -0400 Subject: [PATCH 061/238] black --- bbot/modules/internal/excavate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index f41489e95..9455078cf 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -9,6 +9,7 @@ from bbot.modules.internal.base import BaseInternalModule from urllib.parse import urlparse, urljoin, parse_qs, urlunparse + def find_subclasses(obj, base_class): subclasses = [] for name, member in inspect.getmembers(obj): From dd96e4ce61dc823eb4174489570f388ce30d0861 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 21:49:21 -0400 Subject: [PATCH 062/238] adding doc strings for new excavate functionality --- bbot/modules/internal/excavate.py | 122 +++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 9455078cf..7a83ec52f 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -11,6 +11,26 @@ def find_subclasses(obj, base_class): + """ + Finds and returns subclasses of a specified base class within an object. + + Parameters: + obj : object + The object to inspect for subclasses. + base_class : type + The base class to find subclasses of. + + Returns: + list + A list of subclasses found within the object. + + Example: + >>> class A: pass + >>> class B(A): pass + >>> class C(A): pass + >>> find_subclasses(locals(), A) + [, ] + """ subclasses = [] for name, member in inspect.getmembers(obj): if inspect.isclass(member) and issubclass(member, base_class) and member is not base_class: @@ -19,6 +39,24 @@ def find_subclasses(obj, base_class): def _exclude_key(original_dict, key_to_exclude): + """ + Returns a new dictionary excluding the specified key from the original dictionary. + + Parameters: + original_dict : dict + The dictionary to exclude the key from. + key_to_exclude : hashable + The key to exclude. + + Returns: + dict + A new dictionary without the specified key. + + Example: + >>> original = {'a': 1, 'b': 2, 'c': 3} + >>> _exclude_key(original, 'b') + {'a': 1, 'c': 3} + """ return {key: value for key, value in original_dict.items() if key != key_to_exclude} @@ -72,6 +110,24 @@ def __init__(self, excavate): self.name = "" async def preprocess(self, r, event, discovery_context): + """ + Preprocesses YARA rule results, extracts meta tags, and configures a YaraRuleSettings object. + + This method retrieves optional meta tags from YARA rules and uses them to configure a YaraRuleSettings object. + It formats the results from the YARA engine into a suitable format for the process() method and initiates + a call to process(), passing on the pre-processed YARA results, event data, YARA rule settings, and discovery context. + + Parameters: + r : YaraMatch + The YARA match object containing the rule and meta information. + event : Event + The event data associated with the YARA match. + discovery_context : DiscoveryContext + The context in which the discovery is made. + + Returns: + None + """ description = "" tags = [] emit_match = False @@ -90,7 +146,26 @@ async def preprocess(self, r, event, discovery_context): await self.process(yara_results, event, yara_rule_settings, discovery_context) async def process(self, yara_results, event, yara_rule_settings, discovery_context): - + """ + Processes YARA rule results and reports events with enriched data. + + This method iterates over the provided YARA rule results and constructs event data for each match. + It enriches the event data with host, URL, and description information, and conditionally includes + matched data based on the YaraRuleSettings. Finally, it reports the constructed event data. + + Parameters: + yara_results : dict + A dictionary where keys are YARA rule identifiers and values are lists of matched data strings. + event : Event + The event data associated with the YARA match. + yara_rule_settings : YaraRuleSettings + The settings configured from YARA rule meta tags, including description, tags, and emit_match flag. + discovery_context : DiscoveryContext + The context in which the discovery is made. + + Returns: + None + """ for identifier, results in yara_results.items(): for result in results: event_data = {"host": str(event.host), "url": event.data.get("url", "")} @@ -100,6 +175,25 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte await self.report(event_data, event, yara_rule_settings, discovery_context) async def report_prep(self, event_data, event_type, event, tags): + """ + Prepares an event draft for reporting by creating and tagging the event. + + This method creates an event draft using the provided event data and type, associating it with a parent event. + It tags the event draft with the provided tags and returns the draft. If event creation fails, it returns None. + + Parameters: + event_data : dict + The data to be included in the event. + event_type : str + The type of the event being reported. + event : Event + The parent event to which this event draft is related. + tags : list + A list of tags to be associated with the event draft. + + Returns: + EventDraft or None + """ event_draft = self.excavate.make_event(event_data, event_type, parent=event) if not event_draft: return None @@ -109,6 +203,32 @@ async def report_prep(self, event_data, event_type, event, tags): async def report( self, event_data, event, yara_rule_settings, discovery_context, event_type="FINDING", abort_if=None, **kwargs ): + """ + Reports an event by preparing an event draft and emitting it. + + Processes the provided event data, sets a default description if needed, prepares the event draft, and emits it. + It constructs a context string for the event and uses the report_prep method to create the event draft. If the draft is successfully + created, it emits the event. + + Parameters: + event_data : dict + The data to be included in the event. + event : Event + The parent event to which this event is related. + yara_rule_settings : YaraRuleSettings + The settings configured from YARA rule meta tags, including description and tags. + discovery_context : DiscoveryContext + The context in which the discovery is made. + event_type : str, optional + The type of the event being reported, default is "FINDING". + abort_if : callable, optional + A callable that determines if the event emission should be aborted. + **kwargs : dict + Additional keyword arguments to pass to the report_prep method. + + Returns: + None + """ # If a description is not set and is needed, provide a basic one if event_type == "FINDING" and "description" not in event_data.keys(): From bf1599c65e68806f91f97b89c82fbdb66576d6cb Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 21:55:29 -0400 Subject: [PATCH 063/238] more docstring details --- bbot/modules/internal/excavate.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 7a83ec52f..59462f58a 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -117,6 +117,8 @@ async def preprocess(self, r, event, discovery_context): It formats the results from the YARA engine into a suitable format for the process() method and initiates a call to process(), passing on the pre-processed YARA results, event data, YARA rule settings, and discovery context. + This should typically NOT be overridden. + Parameters: r : YaraMatch The YARA match object containing the rule and meta information. @@ -153,6 +155,8 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte It enriches the event data with host, URL, and description information, and conditionally includes matched data based on the YaraRuleSettings. Finally, it reports the constructed event data. + Override when custom processing and/or validation is needed on data before reporting. + Parameters: yara_results : dict A dictionary where keys are YARA rule identifiers and values are lists of matched data strings. @@ -181,6 +185,8 @@ async def report_prep(self, event_data, event_type, event, tags): This method creates an event draft using the provided event data and type, associating it with a parent event. It tags the event draft with the provided tags and returns the draft. If event creation fails, it returns None. + Override when an event needs to be modified before it is emitted - for example, custom tags need to be conditionally added. + Parameters: event_data : dict The data to be included in the event. @@ -210,6 +216,8 @@ async def report( It constructs a context string for the event and uses the report_prep method to create the event draft. If the draft is successfully created, it emits the event. + Typically not overridden, but might need to be if custom logic is needed to build description/context, etc. + Parameters: event_data : dict The data to be included in the event. From 419d1bb8dd986ea46b10cdc80652a0151955c819 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 23:06:57 -0400 Subject: [PATCH 064/238] adding excavate IP url test --- .../module_tests/test_module_excavate.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 060fafdb7..6f3b99d90 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -349,6 +349,27 @@ def check(self, module_test, events): assert any(e.data == "https://asdffoo.test.notreal/some/path" for e in events) +class TestExcavateURL_IP(TestExcavate): + + targets = ["http://127.0.0.1:8888/", "127.0.0.2"] + + async def setup_before_prep(self, module_test): + module_test.httpserver.expect_request("/").respond_with_data( + "SomeSMooshedDATAhttps://127.0.0.2/some/path" + ) + + def check(self, module_test, events): + print("@@@@") + for e in events: + print(e) + print(e.type) + if e.type == "URL_UNVERIFIED": + print(e.data) + + assert any(e.data == "127.0.0.2" for e in events) + assert any(e.data == "https://127.0.0.2/some/path" for e in events) + + class TestExcavateSerializationNegative(TestExcavate): async def setup_before_prep(self, module_test): module_test.httpserver.expect_request("/").respond_with_data( From 0c965bd3df6d23963a4215bbf4478783ac68c3d2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 23:08:08 -0400 Subject: [PATCH 065/238] adding excavate IP url test (cleanup) --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 6f3b99d90..c2e596099 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -359,13 +359,6 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - print("@@@@") - for e in events: - print(e) - print(e.type) - if e.type == "URL_UNVERIFIED": - print(e.data) - assert any(e.data == "127.0.0.2" for e in events) assert any(e.data == "https://127.0.0.2/some/path" for e in events) From c012129003852621814b29f64b43844245c78b29 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 4 Jul 2024 23:08:54 -0400 Subject: [PATCH 066/238] black --- bbot/test/test_step_2/module_tests/test_module_excavate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index c2e596099..7f7f16269 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -354,9 +354,7 @@ class TestExcavateURL_IP(TestExcavate): targets = ["http://127.0.0.1:8888/", "127.0.0.2"] async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data( - "SomeSMooshedDATAhttps://127.0.0.2/some/path" - ) + module_test.httpserver.expect_request("/").respond_with_data("SomeSMooshedDATAhttps://127.0.0.2/some/path") def check(self, module_test, events): assert any(e.data == "127.0.0.2" for e in events) From 7af7ebff25a62586ce85e0612617520fba80a14b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 5 Jul 2024 14:19:09 -0400 Subject: [PATCH 067/238] merge from 2.0 --- bbot/core/event/base.py | 1 + bbot/modules/internal/excavate.py | 2 +- bbot/modules/lightfuzz_submodules/path.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index df75f9e4c..e7f1d2656 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1075,6 +1075,7 @@ def _data_id(self): data = f"{prefix}:{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" return data + def sanitize_data(self, data): self.parsed_url = self.validators.validate_url_parsed(data) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 59462f58a..a257ca2ea 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -293,7 +293,7 @@ class excavateTestRule(ExcavateRule): } options = { - "retain_querystring": False, + "retain_querystring": True, "yara_max_match_data": 2000, "custom_yara_rules": "", } diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 3f981a3ff..f411d05df 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -44,13 +44,15 @@ async def fuzz(self): http_compare, self.event.data["type"], payloads["doubledot_payload"], cookies ) + self.lightfuzz.debug(f"[POSSIBLE Path Traversal debug] [{path_technique}] DEBUG: singledot_probe URL: [{singledot_probe[3].request.url}] doubledot_probe URL: [{doubledot_probe[3].request.url}]") + if ( singledot_probe[0] == True and doubledot_probe[0] == False and doubledot_probe[3] != None - and not str(doubledot_probe[3].status_code).startswith("4") and doubledot_probe[1] != ["header"] ): + self.results.append( { "type": "FINDING", From f963491b078f19a77d36eac4f876218344944610 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 5 Jul 2024 14:38:24 -0400 Subject: [PATCH 068/238] more merge nonsense --- bbot/core/event/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 946aecc43..df75f9e4c 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1075,10 +1075,6 @@ def _data_id(self): data = f"{prefix}:{self.parsed_url.scheme}:{self.parsed_url.netloc}:{self.parsed_url.path}:{cleaned_query}" return data -<<<<<<< HEAD - -======= ->>>>>>> yara-excavate def sanitize_data(self, data): self.parsed_url = self.validators.validate_url_parsed(data) From 3c09bb62227c58d6e0c57109b6a564223767e543 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 5 Jul 2024 15:40:26 -0400 Subject: [PATCH 069/238] adding lightfuzz tests, moving tests to excavate --- .../module_tests/test_module_excavate.py | 59 ++ .../module_tests/test_module_lightfuzz.py | 986 ++++++++++++++++++ 2 files changed, 1045 insertions(+) create mode 100644 bbot/test/test_step_2/module_tests/test_module_lightfuzz.py diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 7f7f16269..7febb6767 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -729,3 +729,62 @@ def check(self, module_test, events): assert found_url_unverified_spider_max, "Excavate failed to find /spider link" assert found_url_unverified_dummy, "Dummy module did not correctly re-emit" assert found_url_event, "URL was not emitted from non-spider-max URL_UNVERIFIED" + + +class TestExcavate_retain_querystring(ModuleTestBase): + targets = ["http://127.0.0.1:8888/?foo=1"] + modules_overrides = ["httpx", "excavate", "hunt"] + config_overrides = { + "url_querystring_remove": False, + "url_querystring_collapse": False, + "interactsh_disable": True, + "web_spider_depth": 4, + "web_spider_distance": 4, + "modules": { + "excavate": { + "retain_querystring": True, + } + }, + } + + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/", "query_string": "foo=1"} + respond_args = { + "response_data": "alive", + "headers": {"Set-Cookie": "a=b"}, + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + web_parameter_emit = False + for e in events: + if e.type == "WEB_PARAMETER" and "foo" in e.data["url"]: + web_parameter_emit = True + + assert web_parameter_emit + + +class TestExcavate_retain_querystring_not(TestExcavate_retain_querystring): + + config_overrides = { + "url_querystring_remove": False, + "url_querystring_collapse": False, + "interactsh_disable": True, + "web_spider_depth": 4, + "web_spider_distance": 4, + "modules": { + "excavate": { + "retain_querystring": True, + } + }, + } + + def check(self, module_test, events): + web_parameter_emit = False + for e in events: + if e.type == "WEB_PARAMETER" and "foo" not in e.data["url"]: + web_parameter_emit = True + + assert web_parameter_emit + diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py new file mode 100644 index 000000000..3f96aa313 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -0,0 +1,986 @@ +import re + +from .base import ModuleTestBase +from werkzeug.wrappers import Response +from urllib.parse import unquote + + +class Test_Lightfuzz_querystring_noremove(ModuleTestBase): + html_body = 'View detailsView details' + + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] + config_overrides = { + "url_querystring_remove": False, + "interactsh_disable": True, + "web_spider_depth": 4, + "web_spider_distance": 4, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = { + "response_data": self.html_body, + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/product"} + respond_args = { + "response_data": "alive", + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + + web_parameter_emit = False + web_parameter_outofscope = False + + for e in events: + if e.type == "WEB_PARAMETER": + web_parameter_emit = True + + count_url_events = sum(1 for event in events if event.type == "URL") + assert count_url_events == 2 + + +class Test_Lightfuzz_querystring_nocollapse(Test_Lightfuzz_querystring_noremove): + + config_overrides = { + "url_querystring_remove": False, + "url_querystring_collapse": False, + "interactsh_disable": True, + "web_spider_depth": 4, + "web_spider_distance": 4, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + def check(self, module_test, events): + + web_parameter_emit = False + web_parameter_outofscope = False + for e in events: + if e.type == "WEB_PARAMETER": + web_parameter_emit = True + count_url_events = sum(1 for event in events if event.type == "URL") + assert count_url_events == 3 + + +class Test_Lightfuzz_webparameter_outofscope(ModuleTestBase): + + html_body = "" + + targets = ["http://127.0.0.1:8888", "socialmediasite.com"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = { + "response_data": self.html_body, + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + web_parameter_differentsite = False + web_parameter_outofscope = False + + for e in events: + if e.type == "WEB_PARAMETER" and "in-scope" in e.tags and e.host == "socialmediasite.com": + web_parameter_differentsite = True + + if e.type == "WEB_PARAMETER" and e.host == "outofscope.com": + web_parameter_outofscope = True + + assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" + assert not web_parameter_outofscope, "Out of scope domain was emitted" + + +# Path Traversal single dot tolerance +class Test_Lightfuzz_path_singledot(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": False, + "submodule_path": True, + "submodule_ssti": False, + } + }, + } + + async def setup_after_prep(self, module_test): + expect_args = re.compile("/images") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + respond_args = { + "response_data": '"
', + "status": 200, + } + + expect_args = {"method": "GET", "uri": "/"} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + if "filename=" in qs: + value = qs.split("=")[1] + + if "&" in value: + value = value.split("&")[0] + + block = f""" + + + + """ + if value == "%2F.%2Fdefault.jpg" or value == "default.jpg": + return Response(block, status=200) + return Response("file not found", status=500) + + def check(self, module_test, events): + + web_parameter_emitted = False + pathtraversal_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [filename]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + + if ( + "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [single-dot traversal tolerance (url-encoding)]" + in e.data["description"] + ): + pathtraversal_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert pathtraversal_finding_emitted, "Path Traversal single dot tolerance FINDING not emitted" + + +# Path Traversal Absolute path +class Test_Lightfuzz_path_absolute(Test_Lightfuzz_path_singledot): + + etc_passwd = """ +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +sys:x:3:3:sys:/dev:/usr/sbin/nologin +sync:x:4:65534:sync:/bin:/bin/sync +games:x:5:60:games:/usr/games:/usr/sbin/nologin +man:x:6:12:man:/var/cache/man:/usr/sbin/nologin +lp:x:7:7:lp:/var/spool/lpd:/usr/sbin/nologin +""" + + async def setup_after_prep(self, module_test): + + expect_args = {"method": "GET", "uri": "/images", "query_string": "filename=/etc/passwd"} + respond_args = {"response_data": self.etc_passwd} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/images"} + respond_args = {"response_data": "

ERROR: Invalid File

", "status": 200} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = {"method": "GET", "uri": "/"} + respond_args = { + "response_data": '"
', + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + + web_parameter_emitted = False + pathtraversal_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [filename]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [Absolute Path: /etc/passwd]" + in e.data["description"] + ): + pathtraversal_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert pathtraversal_finding_emitted, "Path Traversal single dot tolerance FINDING not emitted" + + +# SSTI Integer Multiplcation +class Test_Lightfuzz_ssti_multiply(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": True, + } + }, + } + + def request_handler(self, request): + qs = str(request.query_string.decode()) + if "data=" in qs: + value = qs.split("=")[1] + if "&" in value: + value = value.split("&")[0] + nums = value.split("%20")[1].split("*") + ints = [int(s) for s in nums] + ssti_block = f"
{str(ints[0] * ints[1])}" + return Response(ssti_block, status=200) + + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = {"response_data": "", "status": 302, "headers": {"Location": "/test?data=1"}} + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + expect_args = re.compile("/test.*") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + + web_parameter_emitted = False + ssti_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [data]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "POSSIBLE Server-side Template Injection. Parameter: [data] Parameter Type: [GETPARAM] Detection Method: [Integer Multiplication]" + in e.data["description"] + ): + ssti_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert ssti_finding_emitted, "SSTI integer multiply FINDING not emitted" + + +# Between Tags XSS Detection +class Test_Lightfuzz_xss(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": True, + "submodule_sqli": False, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + if "search=" in qs: + value = qs.split("=")[1] + if "&" in value: + value = value.split("&")[0] + xss_block = f""" +
+

0 search results for '{unquote(value)}'

+
+
+ """ + return Response(xss_block, status=200) + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + + web_parameter_emitted = False + xss_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags]" in e.data["description"]: + xss_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" + + +# In Tag Attribute XSS Detection +class Test_Lightfuzz_xss_intag(Test_Lightfuzz_xss): + def request_handler(self, request): + qs = str(request.query_string.decode()) + + parameter_block = """ + + Link + + """ + if "foo=" in qs: + value = qs.split("=")[1] + if "&" in value: + value = value.split("&")[0] + + xss_block = f""" +
+
stuff
+
+
+ """ + return Response(xss_block, status=200) + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + expect_args = re.compile("/otherpage.php") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + web_parameter_emitted = False + original_value_captured = False + xss_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [foo]" in e.data["description"]: + web_parameter_emitted = True + if e.data["original_value"] == "bar": + original_value_captured = True + + if e.type == "FINDING": + if "Possible Reflected XSS. Parameter: [foo] Context: [Tag Attribute]" in e.data["description"]: + xss_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert original_value_captured, "original_value not captured" + assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" + + +# In Javascript XSS Detection +class Test_Lightfuzz_xss_injs(Test_Lightfuzz_xss): + def request_handler(self, request): + qs = str(request.query_string.decode()) + parameter_block = """ + + Link + + """ + if "language=" in qs: + value = qs.split("=")[1] + + if "&" in value: + value = value.split("&")[0] + + xss_block = f""" + + + + + +

test

+ + + """ + return Response(xss_block, status=200) + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + expect_args = re.compile("/otherpage.php") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + web_parameter_emitted = False + original_value_captured = False + xss_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [language]" in e.data["description"]: + web_parameter_emitted = True + if e.data["original_value"] == "en": + original_value_captured = True + + if e.type == "FINDING": + if "Possible Reflected XSS. Parameter: [language] Context: [In Javascript]" in e.data["description"]: + xss_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert original_value_captured, "original_value not captured" + assert xss_finding_emitted, "In Javascript XSS FINDING not emitted" + + +# SQLI Single Quote/Two Single Quote (getparam) +class Test_Lightfuzz_sqli(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": True, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + def request_handler(self, request): + qs = str(request.query_string.decode()) + parameter_block = """ +
+
+ + +
+
+ """ + if "search=" in qs: + value = qs.split("=")[1] + + if "&" in value: + value = value.split("&")[0] + + sql_block_normal = f""" +
+

0 search results for '{unquote(value)}'

+
+
+ """ + + sql_block_error = f""" +
+

Found error in SQL query

+
+
+ """ + if value.endswith("'"): + if value.endswith("''"): + return Response(sql_block_normal, status=200) + return Response(sql_block_error, status=500) + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + web_parameter_emitted = False + sqli_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + if e.type == "FINDING": + if ( + "Possible SQL Injection. Parameter: [search] Parameter Type: [GETPARAM] Detection Method: [Single Quote/Two Single Quote]" + in e.data["description"] + ): + sqli_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert sqli_finding_emitted, "SQLi Single/Double Quote getparam FINDING not emitted" + + +# SQLI Single Quote/Two Single Quote (postparam) +class Test_Lightfuzz_sqli_post(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": True, + "submodule_cmdi": False, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + + if "search" in request.form.keys(): + + value = request.form["search"] + + sql_block_normal = f""" +
+

0 search results for '{unquote(value)}'

+
+
+ """ + + sql_block_error = f""" +
+

Found error in SQL query

+
+
+ """ + if value.endswith("'"): + if value.endswith("''"): + return Response(sql_block_normal, status=200) + return Response(sql_block_error, status=500) + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + web_parameter_emitted = False + sqli_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "Possible SQL Injection. Parameter: [search] Parameter Type: [POSTPARAM] Detection Method: [Single Quote/Two Single Quote]" + in e.data["description"] + ): + sqli_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert sqli_finding_emitted, "SQLi Single/Double Quote postparam FINDING not emitted" + + +# SQLI Single Quote/Two Single Quote (headers) +class Test_Lightfuzz_sqli_headers(Test_Lightfuzz_sqli): + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + seed_events = [] + parent_event = module_test.scan.make_event( + "http://127.0.0.1:8888/", + "URL", + module_test.scan.root_event, + module="httpx", + tags=["status-200", "distance-0"], + ) + + data = { + "host": "127.0.0.1", + "type": "HEADER", + "name": "test", + "original_value": None, + "url": "http://127.0.0.1:8888", + "description": "Test Dummy Header", + } + seed_event = module_test.scan.make_event(data, "WEB_PARAMETER", parent_event, tags=["distance-0"]) + seed_events.append(seed_event) + module_test.scan.target.seeds._events = set(seed_events) + + def request_handler(self, request): + + placeholder_block = """ + +

placeholder

+ + """ + + qs = str(request.query_string.decode()) + if request.headers.get("Test") is not None: + header_value = request.headers.get("Test") + + header_block_normal = f""" + +

placeholder

+

test: {header_value}

+ + """ + header_block_error = f""" + +

placeholder

+

Error!

+ + """ + if header_value.endswith("'") and not header_value.endswith("''"): + return Response(header_block_error, status=500) + return Response(header_block_normal, status=200) + return Response(placeholder_block, status=200) + + def check(self, module_test, events): + + web_parameter_emitted = False + sqli_finding_emitted = False + for e in events: + if e.type == "FINDING": + if ( + "Possible SQL Injection. Parameter: [test] Parameter Type: [HEADER] Detection Method: [Single Quote/Two Single Quote]" + in e.data["description"] + ): + sqli_finding_emitted = True + assert sqli_finding_emitted, "SQLi Single/Double Quote headers FINDING not emitted" + + +# SQLI Single Quote/Two Single Quote (cookies) +class Test_Lightfuzz_sqli_cookies(Test_Lightfuzz_sqli): + + async def setup_after_prep(self, module_test): + + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + seed_events = [] + parent_event = module_test.scan.make_event( + "http://127.0.0.1:8888/", + "URL", + module_test.scan.root_event, + module="httpx", + tags=["status-200", "distance-0"], + ) + + data = { + "host": "127.0.0.1", + "type": "COOKIE", + "name": "test", + "original_value": None, + "url": "http://127.0.0.1:8888", + "description": "Test Dummy Header", + } + seed_event = module_test.scan.make_event(data, "WEB_PARAMETER", parent_event, tags=["distance-0"]) + seed_events.append(seed_event) + module_test.scan.target.seeds._events = set(seed_events) + + def request_handler(self, request): + + placeholder_block = """ + +

placeholder

+ + """ + + qs = str(request.query_string.decode()) + if request.cookies.get("test") is not None: + header_value = request.cookies.get("test") + + header_block_normal = f""" + +

placeholder

+

test: {header_value}

+ + """ + + header_block_error = f""" + +

placeholder

+

Error!

+ + """ + if header_value.endswith("'") and not header_value.endswith("''"): + return Response(header_block_error, status=500) + return Response(header_block_normal, status=200) + return Response(placeholder_block, status=200) + + def check(self, module_test, events): + + web_parameter_emitted = False + sqli_finding_emitted = False + for e in events: + if e.type == "FINDING": + if ( + "Possible SQL Injection. Parameter: [test] Parameter Type: [COOKIE] Detection Method: [Single Quote/Two Single Quote]" + in e.data["description"] + ): + sqli_finding_emitted = True + assert sqli_finding_emitted, "SQLi Single/Double Quote cookies FINDING not emitted" + + +# SQLi Delay Probe +class Test_Lightfuzz_sqli_delay(Test_Lightfuzz_sqli): + + def request_handler(self, request): + from time import sleep + + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ + """ + if "search=" in qs: + value = qs.split("=")[1] + + if "&" in value: + value = value.split("&")[0] + + sql_block = f""" +
+

0 search results found

+
+
+ """ + if "'%20AND%20(SLEEP(5))%20AND%20" in value: + sleep(5) + + return Response(sql_block, status=200) + return Response(parameter_block, status=200) + + def check(self, module_test, events): + + web_parameter_emitted = False + sqldelay_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "Possible Blind SQL Injection. Parameter: [search] Parameter Type: [GETPARAM] Detection Method: [Delay Probe (1' AND (SLEEP(5)) AND ')]" + in e.data["description"] + ): + sqldelay_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert sqldelay_finding_emitted, "SQLi Delay FINDING not emitted" + + +# CMDi echo canary +class Test_Lightfuzz_cmdi(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": True, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + if "search=" in qs: + value = qs.split("=")[1] + + if "&" in value: + value = value.split("&")[0] + + if "%26%26%20echo%20" in value: + cmdi_value = value.split("%26%26%20echo%20")[1].split("%20")[0] + else: + cmdi_value = value + cmdi_block = f""" +
+

0 search results for '{unquote(cmdi_value)}'

+
+
+ """ + return Response(cmdi_block, status=200) + + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + + web_parameter_emitted = False + cmdi_echocanary_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "POSSIBLE OS Command Injection. Parameter: [search] Parameter Type: [GETPARAM] Detection Method: [echo canary] CMD Probe Delimeters: [&&]" + in e.data["description"] + ): + cmdi_echocanary_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert cmdi_echocanary_finding_emitted, "echo canary CMDi FINDING not emitted" + + +# CMDi interactsh +class Test_Lightfuzz_cmdi_interactsh(Test_Lightfuzz_cmdi): + + @staticmethod + def extract_subdomain_tag(data): + pattern = r"search=.+%26%26%20nslookup%20(.+)\.fakedomain\.fakeinteractsh.com%20%26%26" + match = re.search(pattern, data) + if match: + return match.group(1) + + config_overrides = { + "interactsh_disable": False, + "modules": { + "lightfuzz": { + "submodule_xss": False, + "submodule_sqli": False, + "submodule_cmdi": True, + "submodule_path": False, + "submodule_ssti": False, + } + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + + if "search=" in qs: + value = qs.split("=")[1] + + if "&" in value: + value = value.split("&")[0] + + subdomain_tag = None + subdomain_tag = self.extract_subdomain_tag(request.full_path) + + if subdomain_tag: + self.interactsh_mock_instance.mock_interaction(subdomain_tag) + return Response(parameter_block, status=200) + + async def setup_before_prep(self, module_test): + self.interactsh_mock_instance = module_test.mock_interactsh("lightfuzz") + + module_test.monkeypatch.setattr( + module_test.scan.helpers, "interactsh", lambda *args, **kwargs: self.interactsh_mock_instance + ) + + async def setup_after_prep(self, module_test): + + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + + web_parameter_emitted = False + cmdi_interacttsh_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "VULNERABILITY": + if ( + "OS Command Injection (OOB Interaction) Type: [GETPARAM] Parameter Name: [search] Probe: [&&]" + in e.data["description"] + ): + cmdi_interacttsh_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert cmdi_interacttsh_finding_emitted, "interactsh CMDi FINDING not emitted" From e0f473864699b414398e405670ac84d58adf705b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 6 Jul 2024 18:17:51 -0400 Subject: [PATCH 070/238] adding target direct URL parameter processing, tests, black, cookie bug-fix --- bbot/core/helpers/async_helpers.py | 2 +- bbot/core/helpers/misc.py | 4 +- bbot/modules/internal/excavate.py | 87 ++++++++++++++----- bbot/modules/lightfuzz.py | 28 +++--- bbot/modules/lightfuzz_submodules/base.py | 2 +- bbot/modules/lightfuzz_submodules/cmdi.py | 2 +- bbot/modules/lightfuzz_submodules/crypto.py | 18 ++-- bbot/modules/lightfuzz_submodules/path.py | 8 +- bbot/modules/lightfuzz_submodules/sqli.py | 4 +- bbot/modules/lightfuzz_submodules/ssti.py | 3 +- bbot/modules/lightfuzz_submodules/xss.py | 3 +- .../module_tests/test_module_excavate.py | 35 +++++++- 12 files changed, 140 insertions(+), 56 deletions(-) diff --git a/bbot/core/helpers/async_helpers.py b/bbot/core/helpers/async_helpers.py index 2152f9bc5..dcc510ee4 100644 --- a/bbot/core/helpers/async_helpers.py +++ b/bbot/core/helpers/async_helpers.py @@ -122,6 +122,6 @@ def generator(): # Start the event loop in a separate thread thread = BBOTThread(target=lambda: asyncio.run(runner()), daemon=True, custom_name="bbot async_to_sync_gen()") thread.start() - + # Return the generator return generator() diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index eecab89ca..03eb7a5cc 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -791,6 +791,7 @@ def recursive_decode(data, max_depth=5): rand_pool = string.ascii_lowercase rand_pool_digits = rand_pool + string.digits + def rand_string(length=10, digits=True, numeric_only=False): """ Generates a random string of specified length. @@ -823,7 +824,6 @@ def rand_string(length=10, digits=True, numeric_only=False): return "".join(random.choice(pool) for _ in range(length)) - def truncate_string(s, n): if len(s) > n: return s[: n - 3] + "..." @@ -2755,6 +2755,8 @@ def calculate_entropy(data): data_len = len(data) entropy = -sum((count / data_len) * math.log2(count / data_len) for count in frequency.values()) return entropy + + top_ports_cache = None diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index a257ca2ea..c1ff4850e 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -60,6 +60,15 @@ def _exclude_key(original_dict, key_to_exclude): return {key: value for key, value in original_dict.items() if key != key_to_exclude} +def extract_params_url(parsed_url): + + params = parse_qs(parsed_url.query) + flat_params = {k: v[0] for k, v in params.items()} + + for p, p_value in flat_params.items(): + yield "GET", parsed_url, p, p_value, "direct_url", _exclude_key(flat_params, p) + + def extract_params_location(location_header_value, original_parsed_url): """ Extracts parameters from a location header, yielding them one at a time. @@ -335,6 +344,7 @@ def url_unparse(self, param_type, parsed_url): querystring = "" else: querystring = parsed_url.query + return urlunparse( ( parsed_url.scheme, @@ -852,6 +862,8 @@ async def setup(self): valid_schemes_filename = self.helpers.wordlist_dir / "valid_url_schemes.txt" self.valid_schemes = set(self.helpers.read_file(valid_schemes_filename)) + self.url_querystring_remove = self.scan.config.get("url_querystring_remove", True) + return True async def search(self, data, event, content_type, discovery_context="HTTP response"): @@ -896,6 +908,35 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon self.hugewarning(f"YARA Rule {rule_name} not found in pre-compiled rules") async def handle_event(self, event): + # Harvest GET parameters from URL, if it came directly from the target, and parameter extraction is enabled + if ( + self.parameter_extraction == True + and self.url_querystring_remove == False + and str(event.parent.parent.module) == "TARGET" + ): + self.debug(f"Processing target URL [{event.url}] for GET parameters") + for ( + method, + parsed_url, + parameter_name, + original_value, + regex_name, + additional_params, + ) in extract_params_url(event.parsed_url): + if self.in_bl(parameter_name) == False: + description = f"HTTP Extracted Parameter [{parameter_name}] (Target URL)" + data = { + "host": parsed_url.hostname, + "type": "GETPARAM", + "name": parameter_name, + "original_value": original_value, + "url": self.url_unparse("GETPARAM", parsed_url), + "description": description, + "additional_params": additional_params, + } + context = f"Excavate parsed a URL directly from the scan target for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) + data = event.data # process response data @@ -908,8 +949,9 @@ async def handle_event(self, event): self.assigned_cookies = {} content_type = None reported_location_header = False + for k, v in headers.items(): - if k.lower() == "set_cookie": + if k.lower() == "set-cookie" and self.parameter_extraction: if "=" not in v: self.debug(f"Cookie found without '=': {v}") continue @@ -950,27 +992,28 @@ async def handle_event(self, event): ) # Try to extract parameters from the redirect URL - for ( - method, - parsed_url, - parameter_name, - original_value, - regex_name, - additional_params, - ) in extract_params_location(v, event.parsed_url): - if self.in_bl(parameter_name) == False: - description = f"HTTP Extracted Parameter [{parameter_name}] (Location Header)" - data = { - "host": parsed_url.hostname, - "type": "GETPARAM", - "name": parameter_name, - "original_value": original_value, - "url": self.url_unparse("GETPARAM", parsed_url), - "description": description, - "additional_params": additional_params, - } - context = f"Excavate parsed a location header for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" - await self.emit_event(data, "WEB_PARAMETER", event, context=context) + if self.parameter_extraction: + for ( + method, + parsed_url, + parameter_name, + original_value, + regex_name, + additional_params, + ) in extract_params_location(v, event.parsed_url): + if self.in_bl(parameter_name) == False: + description = f"HTTP Extracted Parameter [{parameter_name}] (Location Header)" + data = { + "host": parsed_url.hostname, + "type": "GETPARAM", + "name": parameter_name, + "original_value": original_value, + "url": self.url_unparse("GETPARAM", parsed_url), + "description": description, + "additional_params": additional_params, + } + context = f"Excavate parsed a location header for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) else: self.warning("location header found but missing redirect_location in HTTP_RESPONSE") diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 7322252f1..0f1b367f9 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -15,22 +15,26 @@ from .lightfuzz_submodules.ssti import SSTILightfuzz from .lightfuzz_submodules.xss import XSSLightfuzz + class lightfuzz(BaseModule): watched_events = ["URL", "WEB_PARAMETER"] produced_events = ["FINDING", "VULNERABILITY"] flags = ["active", "web-thorough"] submodules = { - "sqli": {"description": "SQL Injection","module": SQLiLightfuzz }, - "cmdi": {"description": "Command Injection","module": CmdILightFuzz }, - "xss": {"description": "Cross-site Scripting","module": XSSLightfuzz }, - "path": {"description": "Path Traversal","module": PathTraversalLightfuzz }, - "ssti": {"description": "Server-side Template Injection","module": SSTILightfuzz }, - "crypto": {"description": "Cryptography Probe","module": CryptoLightfuzz } + "sqli": {"description": "SQL Injection", "module": SQLiLightfuzz}, + "cmdi": {"description": "Command Injection", "module": CmdILightFuzz}, + "xss": {"description": "Cross-site Scripting", "module": XSSLightfuzz}, + "path": {"description": "Path Traversal", "module": PathTraversalLightfuzz}, + "ssti": {"description": "Server-side Template Injection", "module": SSTILightfuzz}, + "crypto": {"description": "Cryptography Probe", "module": CryptoLightfuzz}, } options = {"force_common_headers": False, "enabled_submodules": []} - options_desc = {"force_common_headers": "Force emit commonly exploitable parameters that may be difficult to detect", "enabled_submodules": "A list of submodules to enable. Empty list enabled all modules."} + options_desc = { + "force_common_headers": "Force emit commonly exploitable parameters that may be difficult to detect", + "enabled_submodules": "A list of submodules to enable. Empty list enabled all modules.", + } meta = {"description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner"} common_headers = ["x-forwarded-for", "user-agent"] @@ -71,7 +75,9 @@ async def setup(self): if submodule == "submodule_cmdi" and self.scan.config.get("interactsh_disable", False) == False: try: self.interactsh_instance = self.helpers.interactsh() - self.interactsh_domain = await self.interactsh_instance.register(callback=self.interactsh_callback) + self.interactsh_domain = await self.interactsh_instance.register( + callback=self.interactsh_callback + ) except InteractshError as e: self.warning(f"Interactsh failure: {e}") else: @@ -167,13 +173,11 @@ async def handle_event(self, event): } await self.emit_event(data, "WEB_PARAMETER", event) - elif event.type == "WEB_PARAMETER": for submodule, submodule_dict in self.submodules.items(): if getattr(self, submodule): self.debug(f"Starting {submodule_dict['description']} fuzz()") - await self.run_submodule(submodule_dict['module'], event) - + await self.run_submodule(submodule_dict["module"], event) async def cleanup(self): if self.interactsh_instance: @@ -192,4 +196,4 @@ async def finish(self): for r in await self.interactsh_instance.poll(): await self.interactsh_callback(r) except InteractshError as e: - self.debug(f"Error in interact.sh: {e}") \ No newline at end of file + self.debug(f"Error in interact.sh: {e}") diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index dbdb93d49..34cc06fee 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -89,4 +89,4 @@ async def standard_probe(self, event_type, cookies, probe_value, timeout=10): allow_redirects=False, retries=0, timeout=timeout, - ) \ No newline at end of file + ) diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index c6275f43d..bc012d295 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -3,6 +3,7 @@ import urllib.parse + class CmdILightFuzz(BaseLightfuzz): async def fuzz(self): @@ -77,4 +78,3 @@ async def fuzz(self): await self.standard_probe( self.event.data["type"], cookies, f"{probe_value}{interactsh_probe}", timeout=15 ) - diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 415ccde44..1c3955bda 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -2,6 +2,7 @@ from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote + class CryptoLightfuzz(BaseLightfuzz): @staticmethod @@ -56,13 +57,12 @@ def format_agnostic_decode(input_string): data = str return data, encoding - @staticmethod def format_agnostic_encode(data, encoding): if encoding == "hex": encoded_data = data.hex() elif encoding == "base64": - encoded_data = base64.b64encode(data).decode('utf-8') # base64 encoding returns bytes, decode to string + encoded_data = base64.b64encode(data).decode("utf-8") # base64 encoding returns bytes, decode to string else: raise ValueError("Unsupported encoding type specified") return encoded_data @@ -115,17 +115,19 @@ def cryptanalysis(self, input_string): async def padding_oracle_execute(self, data, encoding, cookies, possible_first_byte=False): if possible_first_byte: - baseline_byte = b'\xFF' + baseline_byte = b"\xFF" starting_pos = 0 else: - baseline_byte = b'\x00' + baseline_byte = b"\x00" starting_pos = 1 baseline = self.compare_baseline(self.event.data["type"], data[:-1] + baseline_byte, cookies) differ_count = 0 - for i in range(starting_pos, starting_pos+254): + for i in range(starting_pos, starting_pos + 254): byte = bytes([i]) - oracle_probe = await self.compare_probe(baseline, self.event.data["type"], self.format_agnostic_encode(data[:-1] + byte, encoding), cookies) + oracle_probe = await self.compare_probe( + baseline, self.event.data["type"], self.format_agnostic_encode(data[:-1] + byte, encoding), cookies + ) if oracle_probe[0] == False and "body" in oracle_probe[1]: differ_count += 1 if i == 1: @@ -145,7 +147,9 @@ async def padding_oracle(self, probe_value, cookies): padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies) if padding_oracle_result == None: self.lightfuzz.hugewarning("ENDED UP IN POSSIBLE_FIRST_BYTE SITUATION") - padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies, possible_first_byte=False) + padding_oracle_result = await self.padding_oracle_execute( + data, encoding, cookies, possible_first_byte=False + ) if padding_oracle_result == True: context = f"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]" diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index f411d05df..bd7d1cb74 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -3,6 +3,7 @@ import urllib.parse + class PathTraversalLightfuzz(BaseLightfuzz): async def fuzz(self): @@ -44,7 +45,9 @@ async def fuzz(self): http_compare, self.event.data["type"], payloads["doubledot_payload"], cookies ) - self.lightfuzz.debug(f"[POSSIBLE Path Traversal debug] [{path_technique}] DEBUG: singledot_probe URL: [{singledot_probe[3].request.url}] doubledot_probe URL: [{doubledot_probe[3].request.url}]") + self.lightfuzz.debug( + f"[POSSIBLE Path Traversal debug] [{path_technique}] DEBUG: singledot_probe URL: [{singledot_probe[3].request.url}] doubledot_probe URL: [{doubledot_probe[3].request.url}]" + ) if ( singledot_probe[0] == True @@ -78,6 +81,3 @@ async def fuzz(self): "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Absolute Path: {path}]", } ) - - - diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index d44da88b3..e40350620 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -3,6 +3,7 @@ import statistics + class SQLiLightfuzz(BaseLightfuzz): expected_delay = 5 @@ -93,6 +94,3 @@ async def fuzz(self): else: self.lightfuzz.debug("Could not get baseline for time-delay tests") - - - diff --git a/bbot/modules/lightfuzz_submodules/ssti.py b/bbot/modules/lightfuzz_submodules/ssti.py index d89cb72d7..9adc23adb 100644 --- a/bbot/modules/lightfuzz_submodules/ssti.py +++ b/bbot/modules/lightfuzz_submodules/ssti.py @@ -1,5 +1,6 @@ from .base import BaseLightfuzz + class SSTILightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) @@ -12,5 +13,3 @@ async def fuzz(self): "description": f"POSSIBLE Server-side Template Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Integer Multiplication]", } ) - - diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 55a1d0110..45e2887af 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -2,6 +2,7 @@ import re + class XSSLightfuzz(BaseLightfuzz): def determine_context(self, html, random_string): between_tags = False @@ -75,4 +76,4 @@ async def fuzz(self): if in_javascript: in_javascript_probe = rf"" - await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") \ No newline at end of file + await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 7febb6767..1c5a3165a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -731,6 +731,40 @@ def check(self, module_test, events): assert found_url_event, "URL was not emitted from non-spider-max URL_UNVERIFIED" +class TestExcavateParameterExtraction_targeturl(ModuleTestBase): + targets = ["http://127.0.0.1:8888/?foo=1"] + modules_overrides = ["httpx", "excavate", "hunt"] + config_overrides = { + "url_querystring_remove": False, + "url_querystring_collapse": False, + "interactsh_disable": True, + "web_spider_depth": 4, + "web_spider_distance": 4, + "modules": { + "excavate": { + "retain_querystring": True, + } + }, + } + + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/", "query_string": "foo=1"} + respond_args = { + "response_data": "alive", + "headers": {"Set-Cookie": "a=b"}, + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + web_parameter_emit = False + for e in events: + if e.type == "WEB_PARAMETER" and "HTTP Extracted Parameter [foo] (Target URL)" in e.data["description"]: + web_parameter_emit = True + + assert web_parameter_emit + + class TestExcavate_retain_querystring(ModuleTestBase): targets = ["http://127.0.0.1:8888/?foo=1"] modules_overrides = ["httpx", "excavate", "hunt"] @@ -787,4 +821,3 @@ def check(self, module_test, events): web_parameter_emit = True assert web_parameter_emit - From 8581b40c4430a8972a4aa6b35a12ab4c0ea619bb Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 6 Jul 2024 23:21:49 -0400 Subject: [PATCH 071/238] various bug fixes / test fixes --- bbot/modules/internal/excavate.py | 12 +- bbot/modules/lightfuzz_submodules/base.py | 2 +- bbot/modules/lightfuzz_submodules/sqli.py | 9 +- .../module_tests/test_module_excavate.py | 39 +++- .../module_tests/test_module_lightfuzz.py | 178 ++---------------- 5 files changed, 67 insertions(+), 173 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index c1ff4850e..b312c1779 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -417,9 +417,9 @@ def extract(self): k: v[0] if isinstance(v, list) and len(v) == 1 else v for k, v in query_strings.items() } for parameter_name, original_value in query_strings_dict.items(): - if original_value == None or original_value == "": - original_value = 1 - yield self.output_type, parameter_name, original_value, parsed_url.path, _exclude_key( + # if original_value == None or original_value == "": + # original_value = 1 + yield self.output_type, parameter_name, original_value, url, _exclude_key( query_strings_dict, parameter_name ) @@ -442,10 +442,12 @@ def extract(self): input_tags = form_content_regex.findall(form_content) for parameter_name, original_value in input_tags: - original_value form_parameters[parameter_name] = original_value for parameter_name, original_value in form_parameters.items(): + # if original_value == None or original_value == "": + # original_value = 1 + yield self.output_type, parameter_name, original_value, form_action, _exclude_key( form_parameters, parameter_name ) @@ -914,7 +916,7 @@ async def handle_event(self, event): and self.url_querystring_remove == False and str(event.parent.parent.module) == "TARGET" ): - self.debug(f"Processing target URL [{event.url}] for GET parameters") + self.debug(f"Processing target URL [{urlunparse(event.parsed_url)}] for GET parameters") for ( method, parsed_url, diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 34cc06fee..001444dbd 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -17,7 +17,7 @@ def compare_baseline(self, event_type, probe, cookies): if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" http_compare = self.lightfuzz.helpers.http_compare( - baseline_url, cookies=cookies, include_cache_buster=True + baseline_url, cookies=cookies, include_cache_buster=False ) elif event_type == "COOKIE": cookies_probe = {self.event.data["name"]: f"{probe}"} diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index e40350620..7fac6eac8 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -39,7 +39,12 @@ async def fuzz(self): double_single_quote = await self.compare_probe( http_compare, self.event.data["type"], f"{probe_value}''", cookies ) - + self.lightfuzz.critical("@@@@@@") + self.lightfuzz.critical(probe_value) + self.lightfuzz.critical(single_quote) + self.lightfuzz.critical(double_single_quote) + self.lightfuzz.critical(single_quote[3].request) + self.lightfuzz.critical(double_single_quote[3].request) if "code" in single_quote[1] and "code" not in double_single_quote[1]: self.results.append( { @@ -48,7 +53,7 @@ async def fuzz(self): } ) except HttpCompareError as e: - self.lightfuzz.debug(e) + self.lightfuzz.critical(e) standard_probe_strings = [ f"'||pg_sleep({str(self.expected_delay)})--", # postgres diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 1c5a3165a..c70b13e14 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -738,8 +738,6 @@ class TestExcavateParameterExtraction_targeturl(ModuleTestBase): "url_querystring_remove": False, "url_querystring_collapse": False, "interactsh_disable": True, - "web_spider_depth": 4, - "web_spider_distance": 4, "modules": { "excavate": { "retain_querystring": True, @@ -751,7 +749,6 @@ async def setup_after_prep(self, module_test): expect_args = {"method": "GET", "uri": "/", "query_string": "foo=1"} respond_args = { "response_data": "alive", - "headers": {"Set-Cookie": "a=b"}, "status": 200, } module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) @@ -821,3 +818,39 @@ def check(self, module_test, events): web_parameter_emit = True assert web_parameter_emit + + + + + +class TestExcavate_webparameter_outofscope(ModuleTestBase): + + html_body = "" + + targets = ["http://127.0.0.1:8888", "socialmediasite.com"] + modules_overrides = ["httpx", "excavate", "hunt"] + config_overrides = { + "interactsh_disable": True + } + + async def setup_after_prep(self, module_test): + expect_args = {"method": "GET", "uri": "/"} + respond_args = { + "response_data": self.html_body, + "status": 200, + } + module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) + + def check(self, module_test, events): + web_parameter_differentsite = False + web_parameter_outofscope = False + + for e in events: + if e.type == "WEB_PARAMETER" and "in-scope" in e.tags and e.host == "socialmediasite.com": + web_parameter_differentsite = True + + if e.type == "WEB_PARAMETER" and e.host == "outofscope.com": + web_parameter_outofscope = True + + assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" + assert not web_parameter_outofscope, "Out of scope domain was emitted" \ No newline at end of file diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 3f96aa313..eb27b0849 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -5,140 +5,16 @@ from urllib.parse import unquote -class Test_Lightfuzz_querystring_noremove(ModuleTestBase): - html_body = 'View detailsView details' - - targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz", "excavate"] - config_overrides = { - "url_querystring_remove": False, - "interactsh_disable": True, - "web_spider_depth": 4, - "web_spider_distance": 4, - "modules": { - "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": False, - } - }, - } - - async def setup_after_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/"} - respond_args = { - "response_data": self.html_body, - "status": 200, - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/product"} - respond_args = { - "response_data": "alive", - "status": 200, - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check(self, module_test, events): - - web_parameter_emit = False - web_parameter_outofscope = False - - for e in events: - if e.type == "WEB_PARAMETER": - web_parameter_emit = True - - count_url_events = sum(1 for event in events if event.type == "URL") - assert count_url_events == 2 - - -class Test_Lightfuzz_querystring_nocollapse(Test_Lightfuzz_querystring_noremove): - - config_overrides = { - "url_querystring_remove": False, - "url_querystring_collapse": False, - "interactsh_disable": True, - "web_spider_depth": 4, - "web_spider_distance": 4, - "modules": { - "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": False, - } - }, - } - - def check(self, module_test, events): - - web_parameter_emit = False - web_parameter_outofscope = False - for e in events: - if e.type == "WEB_PARAMETER": - web_parameter_emit = True - count_url_events = sum(1 for event in events if event.type == "URL") - assert count_url_events == 3 - - -class Test_Lightfuzz_webparameter_outofscope(ModuleTestBase): - - html_body = "" - - targets = ["http://127.0.0.1:8888", "socialmediasite.com"] - modules_overrides = ["httpx", "lightfuzz"] - config_overrides = { - "interactsh_disable": True, - "modules": { - "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": False, - } - }, - } - - async def setup_after_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/"} - respond_args = { - "response_data": self.html_body, - "status": 200, - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check(self, module_test, events): - web_parameter_differentsite = False - web_parameter_outofscope = False - - for e in events: - if e.type == "WEB_PARAMETER" and "in-scope" in e.tags and e.host == "socialmediasite.com": - web_parameter_differentsite = True - - if e.type == "WEB_PARAMETER" and e.host == "outofscope.com": - web_parameter_outofscope = True - - assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" - assert not web_parameter_outofscope, "Out of scope domain was emitted" - # Path Traversal single dot tolerance class Test_Lightfuzz_path_singledot(ModuleTestBase): targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] config_overrides = { "interactsh_disable": True, "modules": { "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": False, - "submodule_path": True, - "submodule_ssti": False, + "enabled_submodules": ["path"], } }, } @@ -185,7 +61,7 @@ def check(self, module_test, events): if e.type == "FINDING": if ( - "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [single-dot traversal tolerance (url-encoding)]" + "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [single-dot traversal tolerance" in e.data["description"] ): pathtraversal_finding_emitted = True @@ -248,16 +124,12 @@ def check(self, module_test, events): # SSTI Integer Multiplcation class Test_Lightfuzz_ssti_multiply(ModuleTestBase): targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] config_overrides = { "interactsh_disable": True, "modules": { "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": True, + "enabled_submodules": ["ssti"], } }, } @@ -304,16 +176,12 @@ def check(self, module_test, events): # Between Tags XSS Detection class Test_Lightfuzz_xss(ModuleTestBase): targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] config_overrides = { "interactsh_disable": True, "modules": { "lightfuzz": { - "submodule_xss": True, - "submodule_sqli": False, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": False, + "enabled_submodules": ["sqli"], } }, } @@ -478,21 +346,19 @@ def check(self, module_test, events): # SQLI Single Quote/Two Single Quote (getparam) class Test_Lightfuzz_sqli(ModuleTestBase): targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] config_overrides = { "interactsh_disable": True, "modules": { "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": True, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": False, + "enabled_submodules": ["sqli"], } }, } def request_handler(self, request): + print("((((") + print(request) qs = str(request.query_string.decode()) parameter_block = """
@@ -554,16 +420,12 @@ def check(self, module_test, events): # SQLI Single Quote/Two Single Quote (postparam) class Test_Lightfuzz_sqli_post(ModuleTestBase): targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] config_overrides = { "interactsh_disable": True, "modules": { "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": True, - "submodule_cmdi": False, - "submodule_path": False, - "submodule_ssti": False, + "enabled_submodules": ["sqli"], } }, } @@ -833,16 +695,12 @@ def check(self, module_test, events): # CMDi echo canary class Test_Lightfuzz_cmdi(ModuleTestBase): targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "lightfuzz"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] config_overrides = { "interactsh_disable": True, "modules": { "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": True, - "submodule_path": False, - "submodule_ssti": False, + "enabled_submodules": ["cmdi"], } }, } @@ -919,11 +777,7 @@ def extract_subdomain_tag(data): "interactsh_disable": False, "modules": { "lightfuzz": { - "submodule_xss": False, - "submodule_sqli": False, - "submodule_cmdi": True, - "submodule_path": False, - "submodule_ssti": False, + "enabled_submodules": ["cmdi"], } }, } From 1c14da643dd45df1557dc8563e20ade183aa8d1a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 08:28:35 -0400 Subject: [PATCH 072/238] adjusting path traversal payloads to reduce false postives --- bbot/modules/lightfuzz_submodules/path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index bd7d1cb74..3e9395c88 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -26,8 +26,8 @@ async def fuzz(self): path_techniques = { "single-dot traversal tolerance (no-encoding)": { - "singledot_payload": f"/./{probe_value}", - "doubledot_payload": f"/../{probe_value}", + "singledot_payload": f"/./a/../{probe_value}", + "doubledot_payload": f"/../a/../{probe_value}", }, "single-dot traversal tolerance (url-encoding)": { "singledot_payload": urllib.parse.quote(f"/./{probe_value}".encode(), safe=""), From 2ca0b18a384d25030d0c0a3bfc691cf2402f2230 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 10:11:09 -0400 Subject: [PATCH 073/238] additional path module signatures --- bbot/modules/lightfuzz_submodules/path.py | 24 ++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 3e9395c88..a9296d0af 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -1,6 +1,7 @@ from .base import BaseLightfuzz from bbot.errors import HttpCompareError +import re import urllib.parse @@ -30,11 +31,28 @@ async def fuzz(self): "doubledot_payload": f"/../a/../{probe_value}", }, "single-dot traversal tolerance (url-encoding)": { - "singledot_payload": urllib.parse.quote(f"/./{probe_value}".encode(), safe=""), - "doubledot_payload": urllib.parse.quote(f"/../{probe_value}".encode(), safe=""), + "singledot_payload": urllib.parse.quote(f"/./a/../{probe_value}".encode(), safe=""), + "doubledot_payload": urllib.parse.quote(f"/../a/../{probe_value}".encode(), safe=""), + }, + "single-dot traversal tolerance (non-recursive stripping)": { + "singledot_payload": f"/...//{probe_value}", + "doubledot_payload": f"/....//....//{probe_value}", + }, + "single-dot traversal tolerance (double url-encoding)": { + "singledot_payload": f"%252f.%252f{probe_value}", + "doubledot_payload": f"%252f..%252f{probe_value}", }, } + linux_path_regex = re.match(r"\/(?:\w+\/?)+\.\w+", probe_value) + if linux_path_regex is not None: + original_path_only = "/".join(probe_value.split("/")[:-1]) + original_filename_only = probe_value.split("/")[-1] + path_techniques["single-dot traversal tolerance (start of path validation)"] = { + "singledot_payload": f"{original_path_only}/./{original_filename_only}", + "doubledot_payload": f"{original_path_only}/../{original_filename_only}" + } + for path_technique, payloads in path_techniques.items(): try: @@ -70,7 +88,7 @@ async def fuzz(self): # Absolute path test - absolute_paths = {r"c:\\windows\\win.ini": "; for 16-bit app support", "/etc/passwd": "daemon:x:"} + absolute_paths = {r"c:\\windows\\win.ini": "; for 16-bit app support", "/etc/passwd": "daemon:x:", "../../../../../etc/passwd%00.png": "daemon:x:"} for path, trigger in absolute_paths.items(): r = await self.standard_probe(self.event.data["type"], cookies, path) From 28f81ef728ceacc23cd3981acf8d47ac76bf9f5a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 10:13:08 -0400 Subject: [PATCH 074/238] removing original value defaults --- bbot/modules/internal/excavate.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index b312c1779..4edd041a9 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -417,8 +417,6 @@ def extract(self): k: v[0] if isinstance(v, list) and len(v) == 1 else v for k, v in query_strings.items() } for parameter_name, original_value in query_strings_dict.items(): - # if original_value == None or original_value == "": - # original_value = 1 yield self.output_type, parameter_name, original_value, url, _exclude_key( query_strings_dict, parameter_name ) @@ -445,9 +443,6 @@ def extract(self): form_parameters[parameter_name] = original_value for parameter_name, original_value in form_parameters.items(): - # if original_value == None or original_value == "": - # original_value = 1 - yield self.output_type, parameter_name, original_value, form_action, _exclude_key( form_parameters, parameter_name ) From cefa78ee6403850b8c94ee623b44d77eebf1655f Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 10:14:06 -0400 Subject: [PATCH 075/238] black --- .../test_step_2/module_tests/test_module_excavate.py | 9 ++------- .../test_step_2/module_tests/test_module_lightfuzz.py | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index c70b13e14..c1a37f59d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -820,18 +820,13 @@ def check(self, module_test, events): assert web_parameter_emit - - - class TestExcavate_webparameter_outofscope(ModuleTestBase): html_body = "" targets = ["http://127.0.0.1:8888", "socialmediasite.com"] modules_overrides = ["httpx", "excavate", "hunt"] - config_overrides = { - "interactsh_disable": True - } + config_overrides = {"interactsh_disable": True} async def setup_after_prep(self, module_test): expect_args = {"method": "GET", "uri": "/"} @@ -853,4 +848,4 @@ def check(self, module_test, events): web_parameter_outofscope = True assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" - assert not web_parameter_outofscope, "Out of scope domain was emitted" \ No newline at end of file + assert not web_parameter_outofscope, "Out of scope domain was emitted" diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index eb27b0849..7e6c5850c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -5,7 +5,6 @@ from urllib.parse import unquote - # Path Traversal single dot tolerance class Test_Lightfuzz_path_singledot(ModuleTestBase): targets = ["http://127.0.0.1:8888"] From 3442e3e1c565617034e1b3998a3d0139ef0e75c8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 10:31:17 -0400 Subject: [PATCH 076/238] sqli module improvements --- bbot/modules/lightfuzz_submodules/sqli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 7fac6eac8..1a6c09cd2 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -1,6 +1,7 @@ from .base import BaseLightfuzz from bbot.errors import HttpCompareError +import urllib.parse import statistics @@ -29,7 +30,9 @@ async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) if "original_value" in self.event.data and self.event.data["original_value"] is not None: - probe_value = self.event.data["original_value"] + probe_value = urllib.parse.quote(self.event.data["original_value"], safe="") + + else: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) From 4fefef1fd45b70535b7656d3f5e35af02c64e373 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 14:27:20 -0400 Subject: [PATCH 077/238] changing additional_parameter behavior --- bbot/modules/lightfuzz_submodules/base.py | 37 +++++++++++++++++++---- bbot/modules/lightfuzz_submodules/path.py | 10 ++++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 001444dbd..0054432b3 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -4,6 +4,17 @@ def __init__(self, lightfuzz, event): self.event = event self.results = [] + def additional_params_process(self, additional_params, additional_params_populate_blank_empty): + if additional_params_populate_blank_empty == False: + return additional_params + new_additional_params = {} + for k, v in additional_params.items(): + if v == "": + new_additional_params[k] = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + else: + new_additional_params[k] = v + return new_additional_params + async def send_probe(self, probe): getparams = {self.event.data["name"]: probe} url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams).geturl() @@ -12,7 +23,7 @@ async def send_probe(self, probe): if r: return r.text - def compare_baseline(self, event_type, probe, cookies): + def compare_baseline(self, event_type, probe, cookies, additional_params_populate_empty=False): if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" @@ -32,13 +43,17 @@ def compare_baseline(self, event_type, probe, cookies): elif event_type == "POSTPARAM": data = {self.event.data["name"]: f"{probe}"} if self.event.data["additional_params"] is not None: - data.update(self.event.data["additional_params"]) + data.update( + self.additional_params_process( + self.event.data["additional_params"], additional_params_populate_empty + ) + ) http_compare = self.lightfuzz.helpers.http_compare( self.event.data["url"], method="POST", include_cache_buster=False, data=data, cookies=cookies ) return http_compare - async def compare_probe(self, http_compare, event_type, probe, cookies): + async def compare_probe(self, http_compare, event_type, probe, cookies, additional_params_populate_empty=False): if event_type == "GETPARAM": probe_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" @@ -52,13 +67,19 @@ async def compare_probe(self, http_compare, event_type, probe, cookies): elif event_type == "POSTPARAM": data = {self.event.data["name"]: f"{probe}"} if self.event.data["additional_params"] is not None: - data.update(self.event.data["additional_params"]) + data.update( + self.additional_params_process( + self.event.data["additional_params"], additional_params_populate_empty + ) + ) compare_result = await http_compare.compare( self.event.data["url"], method="POST", data=data, cookies=cookies ) return compare_result - async def standard_probe(self, event_type, cookies, probe_value, timeout=10): + async def standard_probe( + self, event_type, cookies, probe_value, timeout=10, additional_params_populate_empty=False + ): method = "GET" if event_type == "GETPARAM": @@ -76,7 +97,11 @@ async def standard_probe(self, event_type, cookies, probe_value, timeout=10): method = "POST" data = {self.event.data["name"]: probe_value} if self.event.data["additional_params"] is not None: - data.update(self.event.data["additional_params"]) + data.update( + self.additional_params_process( + self.event.data["additional_params"], additional_params_populate_empty + ) + ) else: data = {} self.lightfuzz.debug(f"standard_probe requested URL: [{url}]") diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index a9296d0af..4b51f357d 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -49,8 +49,8 @@ async def fuzz(self): original_path_only = "/".join(probe_value.split("/")[:-1]) original_filename_only = probe_value.split("/")[-1] path_techniques["single-dot traversal tolerance (start of path validation)"] = { - "singledot_payload": f"{original_path_only}/./{original_filename_only}", - "doubledot_payload": f"{original_path_only}/../{original_filename_only}" + "singledot_payload": f"{original_path_only}/./{original_filename_only}", + "doubledot_payload": f"{original_path_only}/../{original_filename_only}", } for path_technique, payloads in path_techniques.items(): @@ -88,7 +88,11 @@ async def fuzz(self): # Absolute path test - absolute_paths = {r"c:\\windows\\win.ini": "; for 16-bit app support", "/etc/passwd": "daemon:x:", "../../../../../etc/passwd%00.png": "daemon:x:"} + absolute_paths = { + r"c:\\windows\\win.ini": "; for 16-bit app support", + "/etc/passwd": "daemon:x:", + "../../../../../etc/passwd%00.png": "daemon:x:", + } for path, trigger in absolute_paths.items(): r = await self.standard_probe(self.event.data["type"], cookies, path) From 7e973e4f5f473ad3e13252e8c03000b474a48321 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 14:28:09 -0400 Subject: [PATCH 078/238] sqli to use new additional_params logic --- bbot/modules/lightfuzz_submodules/sqli.py | 41 ++++++++++++++++++----- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 1a6c09cd2..4b4c72f3c 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -32,23 +32,42 @@ async def fuzz(self): if "original_value" in self.event.data and self.event.data["original_value"] is not None: probe_value = urllib.parse.quote(self.event.data["original_value"], safe="") - else: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) - http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) + http_compare = self.compare_baseline( + self.event.data["type"], probe_value, cookies, additional_params_populate_empty=True + ) try: - single_quote = await self.compare_probe(http_compare, self.event.data["type"], f"{probe_value}'", cookies) + single_quote = await self.compare_probe( + http_compare, + self.event.data["type"], + f"{probe_value}'", + cookies, + additional_params_populate_empty=True, + ) double_single_quote = await self.compare_probe( - http_compare, self.event.data["type"], f"{probe_value}''", cookies + http_compare, + self.event.data["type"], + f"{probe_value}''", + cookies, + additional_params_populate_empty=True, ) self.lightfuzz.critical("@@@@@@") + self.lightfuzz.critical(self.event.data["name"]) + self.lightfuzz.critical(self.event.data["type"]) self.lightfuzz.critical(probe_value) self.lightfuzz.critical(single_quote) self.lightfuzz.critical(double_single_quote) self.lightfuzz.critical(single_quote[3].request) + self.lightfuzz.critical(single_quote[3].request.content) self.lightfuzz.critical(double_single_quote[3].request) - if "code" in single_quote[1] and "code" not in double_single_quote[1]: + self.lightfuzz.critical(double_single_quote[3].request.content) + + self.lightfuzz.hugeinfo(single_quote[3].status_code) + self.lightfuzz.hugeinfo(double_single_quote[3].status_code) + + if "code" in single_quote[1] and (single_quote[3].status_code != double_single_quote[3].status_code): self.results.append( { "type": "FINDING", @@ -66,8 +85,12 @@ async def fuzz(self): ] method = "GET" - baseline_1 = await self.standard_probe(self.event.data["type"], cookies, probe_value) - baseline_2 = await self.standard_probe(self.event.data["type"], cookies, probe_value) + baseline_1 = await self.standard_probe( + self.event.data["type"], cookies, probe_value, additional_params_populate_empty=True + ) + baseline_2 = await self.standard_probe( + self.event.data["type"], cookies, probe_value, additional_params_populate_empty=True + ) if baseline_1 and baseline_2: baseline_1_delay = baseline_1.elapsed.total_seconds() @@ -77,7 +100,9 @@ async def fuzz(self): for p in standard_probe_strings: confirmations = 0 for i in range(0, 3): - r = await self.standard_probe(self.event.data["type"], cookies, f"{probe_value}{p}") + r = await self.standard_probe( + self.event.data["type"], cookies, f"{probe_value}{p}", additional_params_populate_empty=True + ) if not r: self.lightfuzz.debug("delay measure request failed") break From 9e97933b928a420b07986c4e506bde6181526c7c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 15:20:52 -0400 Subject: [PATCH 079/238] fixing multuple same header extraction --- bbot/core/event/base.py | 6 +- bbot/modules/internal/excavate.py | 65 +++++++++++++++++-- .../module_tests/test_module_excavate.py | 35 ++++++++++ 3 files changed, 98 insertions(+), 8 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index df75f9e4c..fed842455 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1232,7 +1232,11 @@ def sanitize_data(self, data): k, v = i.split(":", 1) k = k.strip().lower() v = v.lstrip() - header_dict[k] = v + if k in header_dict: + header_dict[k].append(v) + else: + header_dict[k] = [v] + data["header-dict"] = header_dict # move URL to the front of the dictionary for visibility data = dict(data) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 4edd041a9..43acefede 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -938,14 +938,15 @@ async def handle_event(self, event): # process response data body = event.data.get("body", "") - headers = event.data.get("header-dict", "") + headers = event.data.get("header-dict", {}) - if body == "" and headers == "": + if body == "" and headers == {}: return self.assigned_cookies = {} content_type = None reported_location_header = False +<<<<<<< HEAD for k, v in headers.items(): if k.lower() == "set-cookie" and self.parameter_extraction: @@ -986,10 +987,60 @@ async def handle_event(self, event): await self.emit_event( url_event, context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', +======= + for header, header_values in headers.items(): + self.critical(header) + for header_value in header_values: + self.hugewarning(header_value) + if header.lower() == "set-cookie": + if "=" not in header_value: + self.debug(f"Cookie found without '=': {header_value}") + continue + else: + cookie_name = header_value.split("=")[0] + cookie_value = header_value.split("=")[1].split(";")[0] + + if self.in_bl(cookie_value) == False: + self.assigned_cookies[cookie_name] = cookie_value + description = f"Set-Cookie Assigned Cookie [{cookie_name}]" + data = { + "host": str(event.host), + "type": "COOKIE", + "name": cookie_name, + "original_value": cookie_value, + "url": self.url_unparse("COOKIE", event.parsed_url), + "description": description, + } + context = f"Excavate noticed a set-cookie header for cookie [{cookie_name}] and emitted a WEB_PARAMETER for it" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) + else: + self.debug(f"blocked cookie parameter [{cookie_name}] due to BL match") + if header.lower() == "location": + redirect_location = getattr(event, "redirect_location", "") + if redirect_location: + scheme = self.helpers.is_uri(redirect_location, return_scheme=True) + if scheme in ("http", "https"): + web_spider_distance = getattr(event, "web_spider_distance", 0) + num_redirects = max(getattr(event, "num_redirects", 0), web_spider_distance) + if num_redirects <= self.scan.web_max_redirects: + # we do not want to allow the web_spider_distance to be incremented on redirects, so we do not add spider-danger tag + url_event = self.make_event( + redirect_location, "URL_UNVERIFIED", event, tags="affiliate" +>>>>>>> bc4483fe (fixing multuple same header extraction) ) + if url_event is not None: + reported_location_header = True + await self.emit_event( + url_event, + context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', + ) +<<<<<<< HEAD # Try to extract parameters from the redirect URL if self.parameter_extraction: +======= + # Try to extract parameters from the redirect URL +>>>>>>> bc4483fe (fixing multuple same header extraction) for ( method, parsed_url, @@ -1012,10 +1063,10 @@ async def handle_event(self, event): context = f"Excavate parsed a location header for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" await self.emit_event(data, "WEB_PARAMETER", event, context=context) - else: - self.warning("location header found but missing redirect_location in HTTP_RESPONSE") - if k.lower() == "content-type": - content_type = headers["content-type"] + else: + self.warning("location header found but missing redirect_location in HTTP_RESPONSE") + if header.lower() == "content-type": + content_type = headers["content-type"][0] await self.search( body, @@ -1029,7 +1080,7 @@ async def handle_event(self, event): # Failure to do so results in a race against the same URL extracted by the URLExtractor submodule # If the extracted URL wins, it will cause the manual one to be a dupe, but it will have a higher web_spider_distance. headers.pop("location") - headers_str = "\n".join(f"{k}: {v}" for k, v in headers.items()) + headers_str = "\n".join(f"{k}: {v}" for k, values in headers.items() for v in values) await self.search( headers_str, diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index c1a37f59d..0540bf397 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -849,3 +849,38 @@ def check(self, module_test, events): assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" assert not web_parameter_outofscope, "Out of scope domain was emitted" + +class TestExcavateHeaders(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["excavate", "httpx"] + config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} + + async def setup_before_prep(self, module_test): + + module_test.httpserver.expect_request("/").respond_with_data( + "

test

", + status=200, + headers={ + "Set-Cookie": [ + "COOKIE1=aaaa; Secure; HttpOnly", + "COOKIE2=bbbb; Secure; HttpOnly; SameSite=None", + ] + }, + ) + + def check(self, module_test, events): + + found_first_cookie = False + found_second_cookie = False + + for e in events: + if e.type == "WEB_PARAMETER": + if e.data["name"] == "COOKIE1": + found_first_cookie = True + if e.data["name"] == "COOKIE2": + found_second_cookie = True + + assert found_first_cookie == True + assert found_second_cookie == True + From 0e44ae6a7e7a6a6ce636c22fcafac44c324722b0 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 15:25:44 -0400 Subject: [PATCH 080/238] remove debug --- bbot/modules/internal/excavate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 43acefede..f0699d4b5 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -989,9 +989,7 @@ async def handle_event(self, event): context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', ======= for header, header_values in headers.items(): - self.critical(header) for header_value in header_values: - self.hugewarning(header_value) if header.lower() == "set-cookie": if "=" not in header_value: self.debug(f"Cookie found without '=': {header_value}") @@ -1048,7 +1046,7 @@ async def handle_event(self, event): original_value, regex_name, additional_params, - ) in extract_params_location(v, event.parsed_url): + ) in extract_params_location(header_value, event.parsed_url): if self.in_bl(parameter_name) == False: description = f"HTTP Extracted Parameter [{parameter_name}] (Location Header)" data = { From 8a837c773ab6ea28146d3d1cb00d0ec18f6a6c1e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 16:47:23 -0400 Subject: [PATCH 081/238] merge from cherry pick --- bbot/modules/internal/excavate.py | 50 ++--------------------- bbot/modules/lightfuzz_submodules/sqli.py | 21 +++------- 2 files changed, 8 insertions(+), 63 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index f0699d4b5..fc424b801 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -941,53 +941,13 @@ async def handle_event(self, event): headers = event.data.get("header-dict", {}) if body == "" and headers == {}: + return self.assigned_cookies = {} content_type = None reported_location_header = False -<<<<<<< HEAD - - for k, v in headers.items(): - if k.lower() == "set-cookie" and self.parameter_extraction: - if "=" not in v: - self.debug(f"Cookie found without '=': {v}") - continue - else: - cookie_name = v.split("=")[0] - cookie_value = v.split("=")[1].split(";")[0] - - if self.in_bl(cookie_value) == False: - self.assigned_cookies[cookie_name] = cookie_value - description = f"Set-Cookie Assigned Cookie [{cookie_name}]" - data = { - "host": str(event.host), - "type": "COOKIE", - "name": cookie_name, - "original_value": cookie_value, - "url": self.url_unparse("COOKIE", event.parsed_url), - "description": description, - } - context = f"Excavate noticed a set-cookie header for cookie [{cookie_name}] and emitted a WEB_PARAMETER for it" - await self.emit_event(data, "WEB_PARAMETER", event, context=context) - else: - self.debug(f"blocked cookie parameter [{cookie_name}] due to BL match") - if k.lower() == "location": - redirect_location = getattr(event, "redirect_location", "") - if redirect_location: - scheme = self.helpers.is_uri(redirect_location, return_scheme=True) - if scheme in ("http", "https"): - web_spider_distance = getattr(event, "web_spider_distance", 0) - num_redirects = max(getattr(event, "num_redirects", 0), web_spider_distance) - if num_redirects <= self.max_redirects: - # we do not want to allow the web_spider_distance to be incremented on redirects, so we do not add spider-danger tag - url_event = self.make_event(redirect_location, "URL_UNVERIFIED", event, tags="affiliate") - if url_event is not None: - reported_location_header = True - await self.emit_event( - url_event, - context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', -======= + for header, header_values in headers.items(): for header_value in header_values: if header.lower() == "set-cookie": @@ -1024,7 +984,6 @@ async def handle_event(self, event): # we do not want to allow the web_spider_distance to be incremented on redirects, so we do not add spider-danger tag url_event = self.make_event( redirect_location, "URL_UNVERIFIED", event, tags="affiliate" ->>>>>>> bc4483fe (fixing multuple same header extraction) ) if url_event is not None: reported_location_header = True @@ -1033,12 +992,9 @@ async def handle_event(self, event): context=f'evcavate looked in "Location" header and found {url_event.type}: {url_event.data}', ) -<<<<<<< HEAD # Try to extract parameters from the redirect URL if self.parameter_extraction: -======= - # Try to extract parameters from the redirect URL ->>>>>>> bc4483fe (fixing multuple same header extraction) + for ( method, parsed_url, diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 4b4c72f3c..8218005b4 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -6,10 +6,10 @@ class SQLiLightfuzz(BaseLightfuzz): - expected_delay = 5 + expected_delay = 10 def evaluate_delay(self, mean_baseline, measured_delay): - margin = 1 + margin = 2 if ( mean_baseline + self.expected_delay - margin <= measured_delay @@ -53,19 +53,7 @@ async def fuzz(self): cookies, additional_params_populate_empty=True, ) - self.lightfuzz.critical("@@@@@@") - self.lightfuzz.critical(self.event.data["name"]) - self.lightfuzz.critical(self.event.data["type"]) - self.lightfuzz.critical(probe_value) - self.lightfuzz.critical(single_quote) - self.lightfuzz.critical(double_single_quote) - self.lightfuzz.critical(single_quote[3].request) - self.lightfuzz.critical(single_quote[3].request.content) - self.lightfuzz.critical(double_single_quote[3].request) - self.lightfuzz.critical(double_single_quote[3].request.content) - - self.lightfuzz.hugeinfo(single_quote[3].status_code) - self.lightfuzz.hugeinfo(double_single_quote[3].status_code) + if "code" in single_quote[1] and (single_quote[3].status_code != double_single_quote[3].status_code): self.results.append( @@ -101,12 +89,13 @@ async def fuzz(self): confirmations = 0 for i in range(0, 3): r = await self.standard_probe( - self.event.data["type"], cookies, f"{probe_value}{p}", additional_params_populate_empty=True + self.event.data["type"], cookies, f"{probe_value}{p}", additional_params_populate_empty=True, timeout=20 ) if not r: self.lightfuzz.debug("delay measure request failed") break + d = r.elapsed.total_seconds() self.lightfuzz.debug(f"measured delay: {str(d)}") if self.evaluate_delay(mean_baseline, d): From 613d512c0aed6aeebac617e4dfcb6149eba85fd2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 17:19:40 -0400 Subject: [PATCH 082/238] blind sqli tweaks --- bbot/modules/lightfuzz_submodules/sqli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 8218005b4..221fe1ee6 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -6,10 +6,10 @@ class SQLiLightfuzz(BaseLightfuzz): - expected_delay = 10 + expected_delay = 5 def evaluate_delay(self, mean_baseline, measured_delay): - margin = 2 + margin = 1.5 if ( mean_baseline + self.expected_delay - margin <= measured_delay From e25e198f5e5f80222009dd2efbfac0f9b2404762 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 17:24:07 -0400 Subject: [PATCH 083/238] black --- bbot/modules/lightfuzz_submodules/sqli.py | 8 +++++--- .../test/test_step_2/module_tests/test_module_excavate.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 221fe1ee6..6ed4ce376 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -54,7 +54,6 @@ async def fuzz(self): additional_params_populate_empty=True, ) - if "code" in single_quote[1] and (single_quote[3].status_code != double_single_quote[3].status_code): self.results.append( { @@ -89,13 +88,16 @@ async def fuzz(self): confirmations = 0 for i in range(0, 3): r = await self.standard_probe( - self.event.data["type"], cookies, f"{probe_value}{p}", additional_params_populate_empty=True, timeout=20 + self.event.data["type"], + cookies, + f"{probe_value}{p}", + additional_params_populate_empty=True, + timeout=20, ) if not r: self.lightfuzz.debug("delay measure request failed") break - d = r.elapsed.total_seconds() self.lightfuzz.debug(f"measured delay: {str(d)}") if self.evaluate_delay(mean_baseline, d): diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 0540bf397..382758018 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -850,6 +850,7 @@ def check(self, module_test, events): assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" assert not web_parameter_outofscope, "Out of scope domain was emitted" + class TestExcavateHeaders(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] @@ -883,4 +884,3 @@ def check(self, module_test, events): assert found_first_cookie == True assert found_second_cookie == True - From eed27de378d26d308ea8dc276e40b2bf4f291eb1 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 21:22:24 -0400 Subject: [PATCH 084/238] fix path test --- .../test_step_2/module_tests/test_module_lightfuzz.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 7e6c5850c..870e6bd49 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -32,6 +32,8 @@ async def setup_after_prep(self, module_test): def request_handler(self, request): qs = str(request.query_string.decode()) + print("!!!!") + print(qs) if "filename=" in qs: value = qs.split("=")[1] @@ -44,7 +46,7 @@ def request_handler(self, request): """ - if value == "%2F.%2Fdefault.jpg" or value == "default.jpg": + if value == "%2F.%2Fa%2F..%2Fdefault.jpg" or value == "default.jpg": return Response(block, status=200) return Response("file not found", status=500) @@ -52,13 +54,16 @@ def check(self, module_test, events): web_parameter_emitted = False pathtraversal_finding_emitted = False + print("@@@@") for e in events: + print(e) + print(e.type) if e.type == "WEB_PARAMETER": if "HTTP Extracted Parameter [filename]" in e.data["description"]: web_parameter_emitted = True if e.type == "FINDING": - + print(e.data["description"]) if ( "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [single-dot traversal tolerance" in e.data["description"] From 132304192ccc7c65993639416ad946c7676b4182 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 21:24:14 -0400 Subject: [PATCH 085/238] cleanup --- .../test_step_2/module_tests/test_module_lightfuzz.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 870e6bd49..dda086af7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -32,9 +32,6 @@ async def setup_after_prep(self, module_test): def request_handler(self, request): qs = str(request.query_string.decode()) - print("!!!!") - print(qs) - if "filename=" in qs: value = qs.split("=")[1] @@ -54,16 +51,13 @@ def check(self, module_test, events): web_parameter_emitted = False pathtraversal_finding_emitted = False - print("@@@@") for e in events: - print(e) - print(e.type) + if e.type == "WEB_PARAMETER": if "HTTP Extracted Parameter [filename]" in e.data["description"]: web_parameter_emitted = True if e.type == "FINDING": - print(e.data["description"]) if ( "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [single-dot traversal tolerance" in e.data["description"] From 672ecfb0a7294284279d9ae94b9bfe77f8833823 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 21:54:49 -0400 Subject: [PATCH 086/238] little bug fixes --- bbot/modules/lightfuzz.py | 2 +- bbot/modules/lightfuzz_submodules/cmdi.py | 3 +-- .../test_step_2/module_tests/test_module_lightfuzz.py | 11 ++++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 0f1b367f9..460464a82 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -72,7 +72,7 @@ async def setup(self): setattr(self, submodule, True) self.hugeinfo(f"Lightfuzz {submodule_dict['description']} Submodule Enabled") - if submodule == "submodule_cmdi" and self.scan.config.get("interactsh_disable", False) == False: + if submodule == "cmdi" and self.scan.config.get("interactsh_disable", False) == False: try: self.interactsh_instance = self.helpers.interactsh() self.interactsh_domain = await self.interactsh_instance.register( diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index bc012d295..a5561841c 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -59,11 +59,10 @@ async def fuzz(self): ) # Blind OS Command Injection + if self.lightfuzz.interactsh_instance: self.lightfuzz.event_dict[self.event.data["url"]] = self.event - for p in cmdi_probe_strings: - subdomain_tag = self.lightfuzz.helpers.rand_string(4, digits=False) self.lightfuzz.interactsh_subdomain_tags[subdomain_tag] = { "event": self.event, diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index dda086af7..c22cd202b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -179,7 +179,7 @@ class Test_Lightfuzz_xss(ModuleTestBase): "interactsh_disable": True, "modules": { "lightfuzz": { - "enabled_submodules": ["sqli"], + "enabled_submodules": ["xss"], } }, } @@ -794,10 +794,6 @@ def request_handler(self, request): """ if "search=" in qs: - value = qs.split("=")[1] - - if "&" in value: - value = value.split("&")[0] subdomain_tag = None subdomain_tag = self.extract_subdomain_tag(request.full_path) @@ -822,12 +818,17 @@ def check(self, module_test, events): web_parameter_emitted = False cmdi_interacttsh_finding_emitted = False + print("@@@@") for e in events: + print(e) + print(e.type) if e.type == "WEB_PARAMETER": + print(e.data["description"]) if "HTTP Extracted Parameter [search]" in e.data["description"]: web_parameter_emitted = True if e.type == "VULNERABILITY": + print(e.data["description"]) if ( "OS Command Injection (OOB Interaction) Type: [GETPARAM] Parameter Name: [search] Probe: [&&]" in e.data["description"] From 7b8a06f73f6567eb71f86fef1299f13ca0dff0d8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 22:53:46 -0400 Subject: [PATCH 087/238] enhancing xss module --- bbot/modules/lightfuzz_submodules/xss.py | 11 ++++++++--- .../test_step_2/module_tests/test_module_lightfuzz.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 45e2887af..be4cb9ba4 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -38,6 +38,8 @@ async def check_probe(self, probe, match, context): "description": f"Possible Reflected XSS. Parameter: [{self.event.data['name']}] Context: [{context}]", } ) + return True + return False async def fuzz(self): lightfuzz_event = self.event.parent @@ -64,10 +66,13 @@ async def fuzz(self): self.lightfuzz.debug( f"determine_context returned: between_tags [{between_tags}], in_tag_attribute [{in_tag_attribute}], in_javascript [{in_javascript}]" ) - + tags = ["z","svg","img"] if between_tags: - between_tags_probe = f"{random_string}" - await self.check_probe(between_tags_probe, between_tags_probe, "Between Tags") + for tag in tags: + between_tags_probe = f"<{tag}>{random_string}" + result = await self.check_probe(between_tags_probe, between_tags_probe, f"Between Tags ({tag}) tag") + if result == True: + continue if in_tag_attribute: in_tag_attribute_probe = f'{random_string}"' diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index c22cd202b..8d9d6fc0a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -224,7 +224,7 @@ def check(self, module_test, events): web_parameter_emitted = True if e.type == "FINDING": - if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags]" in e.data["description"]: + if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags" in e.data["description"]: xss_finding_emitted = True assert web_parameter_emitted, "WEB_PARAMETER was not emitted" From 0cfe17a6f9f1e013b87960bd5e6bd378e55a24a2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 7 Jul 2024 23:04:34 -0400 Subject: [PATCH 088/238] xss message fix --- bbot/modules/lightfuzz_submodules/xss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index be4cb9ba4..12169055c 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -70,7 +70,7 @@ async def fuzz(self): if between_tags: for tag in tags: between_tags_probe = f"<{tag}>{random_string}" - result = await self.check_probe(between_tags_probe, between_tags_probe, f"Between Tags ({tag}) tag") + result = await self.check_probe(between_tags_probe, between_tags_probe, f"Between Tags ({tag} tag)") if result == True: continue From 7ee9f755f404712a9e72ed1547cf20aa6bb6a27e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 8 Jul 2024 18:54:02 -0400 Subject: [PATCH 089/238] fixing lightfuzz presets --- bbot/presets/web/lightfuzz-intense.yml | 1 + bbot/presets/web/lightfuzz.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml index 5f90f95d2..23b5a27f1 100644 --- a/bbot/presets/web/lightfuzz-intense.yml +++ b/bbot/presets/web/lightfuzz-intense.yml @@ -17,4 +17,5 @@ config: modules: lightfuzz: force_common_headers: True + excavate: retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml index 0ccb740d3..f8deb23c5 100644 --- a/bbot/presets/web/lightfuzz.yml +++ b/bbot/presets/web/lightfuzz.yml @@ -12,3 +12,5 @@ config: modules: lightfuzz: submodules_enabled: [cmdi,crypto,sqli,ssti,xss] + excavate: + retain_querystring: True \ No newline at end of file From 0cf868afadfd1dce5380d7bad533286cbd242f08 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 8 Jul 2024 20:59:47 -0400 Subject: [PATCH 090/238] exit after success for xss between tags --- bbot/modules/lightfuzz_submodules/xss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 12169055c..7144b410f 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -72,7 +72,7 @@ async def fuzz(self): between_tags_probe = f"<{tag}>{random_string}" result = await self.check_probe(between_tags_probe, between_tags_probe, f"Between Tags ({tag} tag)") if result == True: - continue + break if in_tag_attribute: in_tag_attribute_probe = f'{random_string}"' From 4c6e8344ae5f80008c4b169237e3b090a515b829 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 9 Jul 2024 20:06:10 -0400 Subject: [PATCH 091/238] better error handling / black --- bbot/modules/lightfuzz_submodules/base.py | 2 ++ bbot/modules/lightfuzz_submodules/xss.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 0054432b3..f4ef92f86 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -25,6 +25,8 @@ async def send_probe(self, probe): def compare_baseline(self, event_type, probe, cookies, additional_params_populate_empty=False): + http_compare = None + if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" http_compare = self.lightfuzz.helpers.http_compare( diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 7144b410f..326790e9b 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -66,7 +66,7 @@ async def fuzz(self): self.lightfuzz.debug( f"determine_context returned: between_tags [{between_tags}], in_tag_attribute [{in_tag_attribute}], in_javascript [{in_javascript}]" ) - tags = ["z","svg","img"] + tags = ["z", "svg", "img"] if between_tags: for tag in tags: between_tags_probe = f"<{tag}>{random_string}" From ef9d9f6ed727918c30eb0c4968744c39206dce01 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 10 Jul 2024 21:32:10 -0400 Subject: [PATCH 092/238] hopefully less false postives --- bbot/modules/lightfuzz_submodules/path.py | 72 ++++++++++++----------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 4b51f357d..6034b969b 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -35,12 +35,12 @@ async def fuzz(self): "doubledot_payload": urllib.parse.quote(f"/../a/../{probe_value}".encode(), safe=""), }, "single-dot traversal tolerance (non-recursive stripping)": { - "singledot_payload": f"/...//{probe_value}", - "doubledot_payload": f"/....//....//{probe_value}", + "singledot_payload": f"/...//a/....//{probe_value}", + "doubledot_payload": f"/....//a/....//{probe_value}", }, "single-dot traversal tolerance (double url-encoding)": { - "singledot_payload": f"%252f.%252f{probe_value}", - "doubledot_payload": f"%252f..%252f{probe_value}", + "singledot_payload": f"%252f.%252fa%252f..%252f{probe_value}", + "doubledot_payload": f"%252f..%252fa%252f..%252f{probe_value}", }, } @@ -54,37 +54,43 @@ async def fuzz(self): } for path_technique, payloads in path_techniques.items(): - - try: - singledot_probe = await self.compare_probe( - http_compare, self.event.data["type"], payloads["singledot_payload"], cookies - ) - doubledot_probe = await self.compare_probe( - http_compare, self.event.data["type"], payloads["doubledot_payload"], cookies - ) - - self.lightfuzz.debug( - f"[POSSIBLE Path Traversal debug] [{path_technique}] DEBUG: singledot_probe URL: [{singledot_probe[3].request.url}] doubledot_probe URL: [{doubledot_probe[3].request.url}]" - ) - - if ( - singledot_probe[0] == True - and doubledot_probe[0] == False - and doubledot_probe[3] != None - and doubledot_probe[1] != ["header"] - ): - - self.results.append( - { - "type": "FINDING", - "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [{path_technique}]", - } + iterations = 4 # one failed detection is tolerated, as long as its not the first run + confirmations = 0 + while iterations > 0: + try: + singledot_probe = await self.compare_probe( + http_compare, self.event.data["type"], payloads["singledot_payload"], cookies + ) + doubledot_probe = await self.compare_probe( + http_compare, self.event.data["type"], payloads["doubledot_payload"], cookies ) - # no need to report both techniques if they both work + + if ( + singledot_probe[0] == True + and doubledot_probe[0] == False + and doubledot_probe[3] != None + and doubledot_probe[1] != ["header"] + ): + + confirmations += 1 + self.lightfuzz.critical(confirmations) + if confirmations > 2: + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [{path_technique}]", + } + ) + # no need to report both techniques if they both work + break + except HttpCompareError as e: + self.lightfuzz.debug(e) + continue + + iterations -= 1 + if confirmations == 0: break - except HttpCompareError as e: - self.lightfuzz.debug(e) - continue + # Absolute path test From 0304400d80728d720216c8dc35dfe9d0175dee92 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 10 Jul 2024 21:36:59 -0400 Subject: [PATCH 093/238] remove debug --- bbot/modules/lightfuzz_submodules/path.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 6034b969b..616de0aed 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -73,7 +73,6 @@ async def fuzz(self): ): confirmations += 1 - self.lightfuzz.critical(confirmations) if confirmations > 2: self.results.append( { From 676eeeed98d264f0682b84c2d54336386114d641 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 11 Jul 2024 17:32:21 -0400 Subject: [PATCH 094/238] changing confirmations logic --- bbot/modules/lightfuzz_submodules/path.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 616de0aed..754a45433 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -21,8 +21,6 @@ async def fuzz(self): ) return - http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) - # Single dot traversal tolerance test path_techniques = { @@ -58,6 +56,9 @@ async def fuzz(self): confirmations = 0 while iterations > 0: try: + + http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) + singledot_probe = await self.compare_probe( http_compare, self.event.data["type"], payloads["singledot_payload"], cookies ) @@ -73,6 +74,7 @@ async def fuzz(self): ): confirmations += 1 + self.lightfuzz.verbose(f"Got possible Path Traversal detection: [{str(confirmations)}] Confirmations") if confirmations > 2: self.results.append( { From 253b2b3c8b9c58acf45e5cdefee73e6aeafc3434 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 13 Jul 2024 14:56:57 -0400 Subject: [PATCH 095/238] fixing missing import --- bbot/core/helpers/misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 39e763829..3e9b5749c 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -8,6 +8,7 @@ import asyncio import logging import ipaddress +import ahocorasick import regex as re import subprocess as sp from pathlib import Path From ed9cea1e8dbbf3d2472d4134af267ab8d6f2174b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 13 Jul 2024 16:49:49 -0400 Subject: [PATCH 096/238] looks like excavate missed the rename --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 4608fc6c4..c56793a29 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -312,7 +312,7 @@ class excavateTestRule(ExcavateRule): } scope_distance_modifier = None - _max_event_handlers = 8 + _module_threads = 8 parameter_blacklist = [ "__VIEWSTATE", From 896ae854db1dae2e2c03583586871b123679440b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 13 Jul 2024 16:51:54 -0400 Subject: [PATCH 097/238] using correct threads parameter --- bbot/modules/lightfuzz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 460464a82..37f2b06e6 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -54,7 +54,7 @@ class lightfuzz(BaseModule): ] in_scope_only = True - max_event_handlers = 4 + _module_threads = 4 async def setup(self): self.event_dict = {} From 79a9b7eb4c15911bd7e9841402ca7612cbb0635b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 13 Jul 2024 20:43:35 -0400 Subject: [PATCH 098/238] fixing endless loop --- bbot/modules/lightfuzz_submodules/path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 754a45433..0bac43afd 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -85,6 +85,7 @@ async def fuzz(self): # no need to report both techniques if they both work break except HttpCompareError as e: + iterations -= 1 self.lightfuzz.debug(e) continue From 328d8e590078d6be60adeb9c03ffa291450cd893 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 13 Jul 2024 21:04:50 -0400 Subject: [PATCH 099/238] removing excessive debug messages --- bbot/core/helpers/diff.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 59ee96567..212a73caa 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -103,11 +103,9 @@ async def _baseline(self): for k, v in ddiff.items(): for x in list(ddiff[k]): - log.debug(f"Added {k} filter for path: {x.path()}") self.ddiff_filters.append(x.path()) self.baseline_json = baseline_1_json - self.baseline_ignore_headers = [ h.lower() for h in [ @@ -158,7 +156,6 @@ def compare_body(self, content_1, content_2): if len(ddiff.keys()) == 0: return True else: - log.debug(ddiff) return False async def compare( From 3f69c8559737a54fd939ab0cfd4e9cb3a31d5c62 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 14 Jul 2024 11:11:02 -0400 Subject: [PATCH 100/238] adjusting confirmations for path modules --- bbot/modules/lightfuzz_submodules/path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 0bac43afd..206da734c 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -52,7 +52,7 @@ async def fuzz(self): } for path_technique, payloads in path_techniques.items(): - iterations = 4 # one failed detection is tolerated, as long as its not the first run + iterations = 5 # one failed detection is tolerated, as long as its not the first run confirmations = 0 while iterations > 0: try: @@ -75,7 +75,7 @@ async def fuzz(self): confirmations += 1 self.lightfuzz.verbose(f"Got possible Path Traversal detection: [{str(confirmations)}] Confirmations") - if confirmations > 2: + if confirmations > 3: self.results.append( { "type": "FINDING", From f4cf8a51edbeff769039733ba03826985cbbc29f Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 14 Jul 2024 12:27:34 -0400 Subject: [PATCH 101/238] handle speculative parameters properly --- bbot/modules/lightfuzz_submodules/base.py | 13 ++++++++++++- bbot/modules/lightfuzz_submodules/path.py | 7 ++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index f4ef92f86..077c2d9ba 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -55,7 +55,18 @@ def compare_baseline(self, event_type, probe, cookies, additional_params_populat ) return http_compare - async def compare_probe(self, http_compare, event_type, probe, cookies, additional_params_populate_empty=False): + async def compare_probe( + self, + http_compare, + event_type, + probe, + cookies, + additional_params_populate_empty=False, + speculative_mode="GETPARAM", + ): + + if event_type == "SPECULATIVE": + event_type = speculative_mode if event_type == "GETPARAM": probe_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 206da734c..5de44082b 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -58,7 +58,7 @@ async def fuzz(self): try: http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) - + singledot_probe = await self.compare_probe( http_compare, self.event.data["type"], payloads["singledot_payload"], cookies ) @@ -74,7 +74,9 @@ async def fuzz(self): ): confirmations += 1 - self.lightfuzz.verbose(f"Got possible Path Traversal detection: [{str(confirmations)}] Confirmations") + self.lightfuzz.verbose( + f"Got possible Path Traversal detection: [{str(confirmations)}] Confirmations" + ) if confirmations > 3: self.results.append( { @@ -93,7 +95,6 @@ async def fuzz(self): if confirmations == 0: break - # Absolute path test absolute_paths = { From a35789e44556bc7ad40446886e09012f686ab0bc Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 14 Jul 2024 13:59:14 -0400 Subject: [PATCH 102/238] more bugs with speculative parameters, tests --- bbot/modules/lightfuzz_submodules/base.py | 20 ++++++- .../module_tests/test_module_lightfuzz.py | 57 ++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 077c2d9ba..c945b6cb5 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -23,10 +23,15 @@ async def send_probe(self, probe): if r: return r.text - def compare_baseline(self, event_type, probe, cookies, additional_params_populate_empty=False): + def compare_baseline( + self, event_type, probe, cookies, additional_params_populate_empty=False, speculative_mode="GETPARAM" + ): http_compare = None + if event_type == "SPECULATIVE": + event_type = speculative_mode + if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" http_compare = self.lightfuzz.helpers.http_compare( @@ -65,6 +70,8 @@ async def compare_probe( speculative_mode="GETPARAM", ): + http_compare = None + if event_type == "SPECULATIVE": event_type = speculative_mode @@ -91,9 +98,18 @@ async def compare_probe( return compare_result async def standard_probe( - self, event_type, cookies, probe_value, timeout=10, additional_params_populate_empty=False + self, + event_type, + cookies, + probe_value, + timeout=10, + additional_params_populate_empty=False, + speculative_mode="GETPARAM", ): + if event_type == "SPECULATIVE": + event_type = speculative_mode + method = "GET" if event_type == "GETPARAM": url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}" diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 8d9d6fc0a..58fd80d79 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -1,6 +1,6 @@ import re -from .base import ModuleTestBase +from .base import ModuleTestBase, tempwordlist from werkzeug.wrappers import Response from urllib.parse import unquote @@ -837,3 +837,58 @@ def check(self, module_test, events): assert web_parameter_emitted, "WEB_PARAMETER was not emitted" assert cmdi_interacttsh_finding_emitted, "interactsh CMDi FINDING not emitted" + + +class Test_Lightfuzz_speculative(ModuleTestBase): + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["httpx", "excavate", "paramminer_getparams", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": {"enabled_submodules": ["xss"]}, + "paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}, + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + parameter_block = """ + { + "search": 1, + "common": 1 + } + """ + if "search=" in qs: + value = qs.split("=")[1] + if "&" in value: + value = value.split("&")[0] + xss_block = f""" +
+

0 search results for '{unquote(value)}'

+
+
+ """ + return Response(xss_block, status=200) + return Response(parameter_block, status=200, headers={"Content-Type": "application/json"}) + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + excavate_json_extraction = False + xss_finding_emitted = False + + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter (speculative from json content) [search]" in e.data["description"]: + excavate_json_extraction = True + + if e.type == "FINDING": + if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags" in e.data["description"]: + xss_finding_emitted = True + + assert excavate_json_extraction, "Excavate failed to extract json parameter" + assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" From def263e7d392d96260fb675b3a0e973d0ad249bd Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 14 Jul 2024 14:12:14 -0400 Subject: [PATCH 103/238] oops --- bbot/modules/lightfuzz_submodules/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index c945b6cb5..c8486c6b7 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -70,8 +70,6 @@ async def compare_probe( speculative_mode="GETPARAM", ): - http_compare = None - if event_type == "SPECULATIVE": event_type = speculative_mode From a262413ac4286eb398a9959bb120030d01b6ba81 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 20 Jul 2024 11:01:22 -0400 Subject: [PATCH 104/238] custom cookies / new crypto tests / lots of bug fixes --- bbot/core/helpers/web/client.py | 7 + bbot/core/helpers/web/engine.py | 3 + bbot/core/helpers/web/web.py | 12 +- bbot/modules/internal/excavate.py | 36 +++++ bbot/modules/lightfuzz_submodules/base.py | 18 ++- bbot/modules/lightfuzz_submodules/crypto.py | 162 ++++++++++++++------ bbot/scanner/preset/args.py | 27 ++++ bbot/scanner/scanner.py | 7 + 8 files changed, 219 insertions(+), 53 deletions(-) diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py index cd925730d..e84e7fe50 100644 --- a/bbot/core/helpers/web/client.py +++ b/bbot/core/helpers/web/client.py @@ -63,11 +63,18 @@ def __init__(self, *args, **kwargs): headers = kwargs.get("headers", None) if headers is None: headers = {} + + # cookies + cookies = kwargs.get("cookies", None) + if cookies is None: + cookies = {} + # user agent user_agent = self._web_config.get("user_agent", "BBOT") if "User-Agent" not in headers: headers["User-Agent"] = user_agent kwargs["headers"] = headers + kwargs["cookies"] = cookies # proxy proxies = self._web_config.get("http_proxy", None) kwargs["proxies"] = proxies diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py index 8249eb33f..3bfd66623 100644 --- a/bbot/core/helpers/web/engine.py +++ b/bbot/core/helpers/web/engine.py @@ -25,6 +25,7 @@ class HTTPEngine(EngineServer): client_only_options = ( "retries", "max_redirects", + "cookies", ) def __init__(self, socket_path, target, config={}): @@ -42,6 +43,7 @@ def AsyncClient(self, *args, **kwargs): return BBOTAsyncClient.from_config(self.config, self.target, *args, **kwargs) async def request(self, *args, **kwargs): + raise_error = kwargs.pop("raise_error", False) # TODO: use this cache_for = kwargs.pop("cache_for", None) # noqa @@ -66,6 +68,7 @@ async def request(self, *args, **kwargs): client_kwargs = {} for k in list(kwargs): if k in self.client_only_options: + v = kwargs.pop(k) client_kwargs[k] = v diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 4f38c3388..8f0948029 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -350,6 +350,7 @@ async def curl(self, *args, **kwargs): curl_command.append("-k") headers = kwargs.get("headers", {}) + cookies = kwargs.get("cookies", {}) ignore_bbot_global_settings = kwargs.get("ignore_bbot_global_settings", False) @@ -362,10 +363,17 @@ async def curl(self, *args, **kwargs): if "User-Agent" not in headers: headers["User-Agent"] = user_agent - # only add custom headers if the URL is in-scope + # only add custom headers / cookies if the URL is in-scope if self.parent_helper.preset.in_scope(url): for hk, hv in self.web_config.get("http_headers", {}).items(): - headers[hk] = hv + # Only add the header if it doesn't already exist in the headers dictionary + if hk not in headers: + headers[hk] = hv + + for ck, cv in self._web_config.get("http_cookies", {}).items(): + # don't clobber cookies + if ck not in cookies: + cookies[ck] = cv # add the timeout if not "timeout" in kwargs: diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index c56793a29..94d0b310a 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -925,6 +925,42 @@ async def handle_event(self, event): context = f"Excavate parsed a URL directly from the scan target for parameters and found [GETPARAM] Parameter Name: [{parameter_name}] and emitted a WEB_PARAMETER for it" await self.emit_event(data, "WEB_PARAMETER", event, context=context) + # If parameter_extraction is enabled and we assigned custom headers, emit them as WEB_PARAMETER + if self.parameter_extraction == True and self.parent_helper.preset.in_scope(url): + + custom_cookies = self.scan.web_config.get("http_cookies", {}) + for custom_cookie_name, custom_cookie_value in custom_cookies.items(): + description = f"HTTP Extracted Parameter [{custom_cookie_name}] (Custom Cookie)" + data = { + "host": event.parsed_url.hostname, + "type": "COOKIE", + "name": custom_cookie_name, + "original_value": custom_cookie_value, + "url": self.url_unparse("COOKIE", event.parsed_url), + "description": description, + "additional_params": _exclude_key(custom_cookies, custom_cookie_name), + } + context = f"Excavate saw a custom cookie set [{custom_cookie_name}], and emitted a WEB_PARAMETER for it" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) + + + custom_headers = self.scan.web_config.get("http_headers", {}) + for custom_header_name, custom_header_value in custom_headers.items(): + self.hugewarning(custom_header_name) + self.hugewarning(custom_header_value) + description = f"HTTP Extracted Parameter [{custom_header_name}] (Custom Header)" + data = { + "host": event.parsed_url.hostname, + "type": "HEADER", + "name": custom_header_name, + "original_value": custom_header_value, + "url": self.url_unparse("HEADER", event.parsed_url), + "description": description, + "additional_params": _exclude_key(custom_headers, custom_header_name), + } + context = f"Excavate saw a custom header set [{custom_header_name}], and emitted a WEB_PARAMETER for it" + await self.emit_event(data, "WEB_PARAMETER", event, context=context) + data = event.data # process response data diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index c8486c6b7..5e34a2ca7 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -1,3 +1,5 @@ +import copy + class BaseLightfuzz: def __init__(self, lightfuzz, event): self.lightfuzz = lightfuzz @@ -34,6 +36,8 @@ def compare_baseline( if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" + if self.event.data["additional_params"] is not None: + baseline_url = self.lightfuzz.helpers.add_get_params(baseline_url,self.event.data["additional_params"]).geturl() http_compare = self.lightfuzz.helpers.http_compare( baseline_url, cookies=cookies, include_cache_buster=False ) @@ -67,14 +71,22 @@ async def compare_probe( probe, cookies, additional_params_populate_empty=False, + additional_params_override = {}, speculative_mode="GETPARAM", ): + additional_params = copy.deepcopy(self.event.data["additional_params"]) + if additional_params_override: + for k, v in additional_params_override.items(): + additional_params[k] = v + if event_type == "SPECULATIVE": event_type = speculative_mode if event_type == "GETPARAM": probe_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" + if additional_params is not None: + probe_url = self.lightfuzz.helpers.add_get_params(probe_url,additional_params).geturl() compare_result = await http_compare.compare(probe_url, cookies=cookies) elif event_type == "COOKIE": cookies_probe = {self.event.data["name"]: probe} @@ -84,10 +96,10 @@ async def compare_probe( compare_result = await http_compare.compare(self.event.data["url"], headers=headers, cookies=cookies) elif event_type == "POSTPARAM": data = {self.event.data["name"]: f"{probe}"} - if self.event.data["additional_params"] is not None: + if additional_params is not None: data.update( self.additional_params_process( - self.event.data["additional_params"], additional_params_populate_empty + additional_params, additional_params_populate_empty ) ) compare_result = await http_compare.compare( @@ -111,6 +123,8 @@ async def standard_probe( method = "GET" if event_type == "GETPARAM": url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}" + if self.event.data["additional_params"] is not None: + url = self.lightfuzz.helpers.add_get_params(url,self.event.data["additional_params"]).geturl() else: url = self.event.data["url"] if event_type == "COOKIE": diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 1c3955bda..9d7ead946 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -1,6 +1,7 @@ +import base64 +import hashlib from .base import BaseLightfuzz - -from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote +from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote, quote class CryptoLightfuzz(BaseLightfuzz): @@ -19,6 +20,7 @@ def is_base64(s): if base64.b64encode(base64.b64decode(s)).decode() == s: return True except Exception: + return False return False @@ -58,13 +60,16 @@ def format_agnostic_decode(input_string): return data, encoding @staticmethod - def format_agnostic_encode(data, encoding): + def format_agnostic_encode(data, encoding, urlencode=True): if encoding == "hex": encoded_data = data.hex() elif encoding == "base64": encoded_data = base64.b64encode(data).decode("utf-8") # base64 encoding returns bytes, decode to string else: raise ValueError("Unsupported encoding type specified") + + if urlencode: + return quote(encoded_data) return encoded_data @staticmethod @@ -98,12 +103,11 @@ def modify_string(input_string, action="truncate", position=None, extension_leng raise ValueError("Unsupported action") return CryptoLightfuzz.format_agnostic_encode(modified_data, encoding) - def is_likely_encrypted(self, data, threshold=5.5): + def is_likely_encrypted(self, data, threshold=4.5): entropy = self.lightfuzz.helpers.calculate_entropy(data) return entropy >= threshold def cryptanalysis(self, input_string): - likely_crypto = False possible_block_cipher = False data, encoding = self.format_agnostic_decode(input_string) @@ -113,43 +117,61 @@ def cryptanalysis(self, input_string): possible_block_cipher = True return likely_crypto, possible_block_cipher - async def padding_oracle_execute(self, data, encoding, cookies, possible_first_byte=False): - if possible_first_byte: - baseline_byte = b"\xFF" - starting_pos = 0 - else: - baseline_byte = b"\x00" - starting_pos = 1 - - baseline = self.compare_baseline(self.event.data["type"], data[:-1] + baseline_byte, cookies) - differ_count = 0 - for i in range(starting_pos, starting_pos + 254): - byte = bytes([i]) - oracle_probe = await self.compare_probe( - baseline, self.event.data["type"], self.format_agnostic_encode(data[:-1] + byte, encoding), cookies - ) - if oracle_probe[0] == False and "body" in oracle_probe[1]: - differ_count += 1 - if i == 1: - possible_first_byte = True - continue - elif i == 2 and possible_first_byte == True: - # Thats two results which appear "different". Its entirely possible \x00 was the correct padding. We will break from this loop and redo it with the last byte as the baseline instead of the first - return None - if differ_count == 1: - return True - else: - return False + @staticmethod + def possible_block_sizes(ciphertext_length): + potential_block_sizes = [8, 16] + possible_sizes = [] + for block_size in potential_block_sizes: + num_blocks = ciphertext_length / block_size + if num_blocks.is_integer() and num_blocks >= 2: + possible_sizes.append(block_size) + return possible_sizes + + + async def padding_oracle_execute(self, original_data, encoding, cookies, possible_first_byte=False): + + possible_block_sizes = self.possible_block_sizes(len(original_data)) + for block_size in possible_block_sizes: + + ivblock = b"\x00" * block_size + datablock = b"\x00" * block_size + if possible_first_byte: + baseline_byte = b"\xFF" + starting_pos = 0 + else: + baseline_byte = b"\x00" + starting_pos = 1 + + baseline = self.compare_baseline(self.event.data["type"], ivblock[:-1] + baseline_byte + datablock, cookies) + differ_count = 0 + for i in range(starting_pos, starting_pos + 254): + + byte = bytes([i]) + oracle_probe = await self.compare_probe( + baseline, self.event.data["type"], self.format_agnostic_encode(ivblock[:-1] + byte + datablock, encoding), cookies + ) + + if oracle_probe[0] == False and "body" in oracle_probe[1]: + differ_count += 1 + if i == 1: + possible_first_byte = True + continue + elif i == 2 and possible_first_byte == True: + # Thats two results which appear "different". Its entirely possible \x00 was the correct padding. We will break from this loop and redo it with the last byte as the baseline instead of the first + return False, None + if differ_count == 1: + return True, block_size + else: + continue + return False, None async def padding_oracle(self, probe_value, cookies): data, encoding = self.format_agnostic_decode(probe_value) - padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies) + padding_oracle_result, block_size = await self.padding_oracle_execute(data, encoding, cookies) if padding_oracle_result == None: - self.lightfuzz.hugewarning("ENDED UP IN POSSIBLE_FIRST_BYTE SITUATION") - padding_oracle_result = await self.padding_oracle_execute( - data, encoding, cookies, possible_first_byte=False - ) + self.lightfuzz.debug("ended up in possible_first_byte situation - retrying with different first byte") + padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies, possible_first_byte=True) if padding_oracle_result == True: context = f"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]" @@ -157,7 +179,7 @@ async def padding_oracle(self, probe_value, cookies): { "type": "VULNERABILITY", "severity": "HIGH", - "description": f"Padding Oracle Vulnerability. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]", + "description": f"Padding Oracle Vulnerability. Block size: [{str(block_size)}] Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]", "context": context, } ) @@ -182,6 +204,20 @@ async def error_string_search(self, text_dict): } ) + @staticmethod + def identify_hash_function(hash_bytes): + hash_length = len(hash_bytes) + hash_functions = { + 16: hashlib.md5, + 20: hashlib.sha1, + 32: hashlib.sha256, + 48: hashlib.sha384, + 64: hashlib.sha512 + } + + if hash_length in hash_functions: + return hash_functions[hash_length] + async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) if ( @@ -191,20 +227,20 @@ async def fuzz(self): ): probe_value = self.event.data["original_value"] else: - self.lightfuzz.critical( + self.lightfuzz.debug( f"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" ) return - try: truncate_probe_value = self.modify_string(probe_value, action="truncate") mutate_probe_value = self.modify_string(probe_value, action="mutate") except ValueError as e: - self.lightfuzz.critical(f"Encountered error modifying value: {e}, aborting") + self.lightfuzz.debug(f"Encountered error modifying value for parameter {self.event.data['name']}: {e} , aborting") return # Basic crypanalysis - likely_crypto, possible_block_cipher = self.cryptanalysis(probe_value) + likely_crypto, possible_block_cipher = self.cryptanalysis(unquote(probe_value)) + if not likely_crypto: self.lightfuzz.debug("Parameter value does not appear to be cryptographic, aborting tests") return @@ -216,13 +252,14 @@ async def fuzz(self): truncate_probe = await self.compare_probe(http_compare, self.event.data["type"], truncate_probe_value, cookies) mutate_probe = await self.compare_probe(http_compare, self.event.data["type"], mutate_probe_value, cookies) + confirmed_techniques = [] - if mutate_probe[0] == False and "body" in mutate_probe[1]: - if http_compare.compare_body(mutate_probe[3].text, arbitrary_probe[3].text) == False: + if (mutate_probe[0] == False and "body" in mutate_probe[1]): + if (http_compare.compare_body(mutate_probe[3].text, arbitrary_probe[3].text) == False) or mutate_probe[3].text == "": confirmed_techniques.append("Single-byte Mutation") if mutate_probe[0] == False and "body" in mutate_probe[1]: - if http_compare.compare_body(truncate_probe[3].text, arbitrary_probe[3].text) == False: + if (http_compare.compare_body(truncate_probe[3].text, arbitrary_probe[3].text) == False) or truncate_probe[3].text == "": confirmed_techniques.append("Data Truncation") if confirmed_techniques: @@ -230,7 +267,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"Probable Cryptographic Parameter [{self.event.data['name']}] Detection Technique(s): [{','.join(confirmed_techniques)}]", + "description": f"Probable Cryptographic Parameter [{self.event.data['name']}] Parameter Type [{self.event.data['type']}] Detection Technique(s): [{', '.join(confirmed_techniques)}]", "context": context, } ) @@ -240,6 +277,33 @@ async def fuzz(self): {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text} ) - # Padding Oracle Test - if possible_block_cipher: - await self.padding_oracle(probe_value, cookies) + + if confirmed_techniques: + + # Padding Oracle Test + + if possible_block_cipher: + self.lightfuzz.debug("Attempting padding oracle exploit since it looks like a block cipher and we have confirmed crypto") + await self.padding_oracle(probe_value, cookies) + + + # Hash identification / Potential Length extension attack + + data, encoding = CryptoLightfuzz.format_agnostic_decode(probe_value) + hash_function = self.identify_hash_function(data) + hash_instance = hash_function() + # if there are any hash functions which match the length, we check the additional parameters to see if they cause identical changes + # this would indicate they are being used to generate the hash + if hash_function and self.event.data["additional_params"]: + for additional_param_name, additional_param_value in self.event.data["additional_params"].items(): + additional_param_probe = await self.compare_probe(http_compare, self.event.data["type"], probe_value, cookies, additional_params_override={additional_param_name: additional_param_value + "A"}) + # the additional parameter affects the potential hash parameter (suggesting its calculated in the hash) + if additional_param_probe[0] == False and (additional_param_probe[1] == mutate_probe[1]): + context = f"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) that is a likely a hash, which is connected to another parameter {additional_param_name})" + self.results.append( + { + "type": "FINDING", + "description": f"Possible {self.event.data['type']} parameter with {hash_instance.name.upper()} Hash as value. Parameter: [{self.event.data['name']}], linked to additional parameter [{additional_param_name}]", + "context": context, + } + ) diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 9886cae4a..bde9a2f8b 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -149,6 +149,9 @@ def preset_from_args(self): if self.parsed.custom_headers: args_preset.core.merge_custom({"web": {"http_headers": self.parsed.custom_headers}}) + if self.parsed.custom_cookies: + args_preset.core.merge_custom({"web": {"http_cookies": self.parsed.custom_cookies}}) + if self.parsed.custom_yara_rules: args_preset.core.merge_custom( {"modules": {"excavate": {"custom_yara_rules": self.parsed.custom_yara_rules}}} @@ -314,6 +317,13 @@ def create_parser(self, *args, **kwargs): default=[], help="List of custom headers as key value pairs (header=value).", ) + misc.add_argument( + "-C", + "--custom-cookies", + nargs="+", + default=[], + help="List of custom cookies as key value pairs (cookie=value).", + ) misc.add_argument("--custom-yara-rules", "-cy", help="Add custom yara rules to excavate") return p @@ -356,6 +366,23 @@ def sanitize_args(self): custom_headers_dict[k] = v self.parsed.custom_headers = custom_headers_dict + # Custom Cookie Parsing / Validation + custom_cookies_dict = {} + custom_cookie_example = "Example: --custom-cookies foo=bar foo2=bar2" + + for i in self.parsed.custom_cookies: + parts = i.split("=", 1) + if len(parts) != 2: + raise ValidationError(f"Custom cookies not formatted correctly (missing '='). {custom_cookie_example}") + k, v = parts + if not k or not v: + raise ValidationError( + f"Custom cookies not formatted correctly (missing cookie name or value). {custom_cookie_example}" + ) + custom_cookies_dict[k] = v + self.parsed.custom_cookies = custom_cookies_dict + + def validate(self): # validate config options sentinel = object() diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 95bb8238e..50618b12a 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -200,6 +200,13 @@ def __init__( self.warning( "You have enabled custom HTTP headers. These will be attached to all in-scope requests and all requests made by httpx." ) + # custom HTTP cookies warning + self.custom_http_cookies = self.web_config.get("http_cookies", {}) + if self.custom_http_cookies: + self.warning( + "You have enabled custom HTTP cookies. These will be attached to all in-scope requests and all requests made by httpx." + ) + # url file extensions self.url_extension_blacklist = set(e.lower() for e in self.config.get("url_extension_blacklist", [])) From b372166e40d37fc61bc93124ac1a589776e7a111 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 20 Jul 2024 11:24:48 -0400 Subject: [PATCH 105/238] little bug fixes --- bbot/modules/internal/excavate.py | 2 +- bbot/modules/lightfuzz_submodules/crypto.py | 33 +++++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 94d0b310a..cf5dacae3 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -926,7 +926,7 @@ async def handle_event(self, event): await self.emit_event(data, "WEB_PARAMETER", event, context=context) # If parameter_extraction is enabled and we assigned custom headers, emit them as WEB_PARAMETER - if self.parameter_extraction == True and self.parent_helper.preset.in_scope(url): + if self.parameter_extraction == True: custom_cookies = self.scan.web_config.get("http_cookies", {}) for custom_cookie_name, custom_cookie_value in custom_cookies.items(): diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 9d7ead946..3c395b38f 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -291,19 +291,20 @@ async def fuzz(self): data, encoding = CryptoLightfuzz.format_agnostic_decode(probe_value) hash_function = self.identify_hash_function(data) - hash_instance = hash_function() - # if there are any hash functions which match the length, we check the additional parameters to see if they cause identical changes - # this would indicate they are being used to generate the hash - if hash_function and self.event.data["additional_params"]: - for additional_param_name, additional_param_value in self.event.data["additional_params"].items(): - additional_param_probe = await self.compare_probe(http_compare, self.event.data["type"], probe_value, cookies, additional_params_override={additional_param_name: additional_param_value + "A"}) - # the additional parameter affects the potential hash parameter (suggesting its calculated in the hash) - if additional_param_probe[0] == False and (additional_param_probe[1] == mutate_probe[1]): - context = f"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) that is a likely a hash, which is connected to another parameter {additional_param_name})" - self.results.append( - { - "type": "FINDING", - "description": f"Possible {self.event.data['type']} parameter with {hash_instance.name.upper()} Hash as value. Parameter: [{self.event.data['name']}], linked to additional parameter [{additional_param_name}]", - "context": context, - } - ) + if hash_function: + hash_instance = hash_function() + # if there are any hash functions which match the length, we check the additional parameters to see if they cause identical changes + # this would indicate they are being used to generate the hash + if hash_function and self.event.data["additional_params"]: + for additional_param_name, additional_param_value in self.event.data["additional_params"].items(): + additional_param_probe = await self.compare_probe(http_compare, self.event.data["type"], probe_value, cookies, additional_params_override={additional_param_name: additional_param_value + "A"}) + # the additional parameter affects the potential hash parameter (suggesting its calculated in the hash) + if additional_param_probe[0] == False and (additional_param_probe[1] == mutate_probe[1]): + context = f"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) that is a likely a hash, which is connected to another parameter {additional_param_name})" + self.results.append( + { + "type": "FINDING", + "description": f"Possible {self.event.data['type']} parameter with {hash_instance.name.upper()} Hash as value. Parameter: [{self.event.data['name']}], linked to additional parameter [{additional_param_name}]", + "context": context, + } + ) From c7568d26dc9f253d4b00b9d3efb356752fffc04b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 20 Jul 2024 19:44:07 -0400 Subject: [PATCH 106/238] minor bug fix --- bbot/modules/lightfuzz_submodules/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 5e34a2ca7..938869770 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -75,7 +75,7 @@ async def compare_probe( speculative_mode="GETPARAM", ): - additional_params = copy.deepcopy(self.event.data["additional_params"]) + additional_params = copy.deepcopy(self.event.data.get("additional_params", {})) if additional_params_override: for k, v in additional_params_override.items(): additional_params[k] = v @@ -85,7 +85,7 @@ async def compare_probe( if event_type == "GETPARAM": probe_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" - if additional_params is not None: + if additional_params: probe_url = self.lightfuzz.helpers.add_get_params(probe_url,additional_params).geturl() compare_result = await http_compare.compare(probe_url, cookies=cookies) elif event_type == "COOKIE": @@ -96,7 +96,7 @@ async def compare_probe( compare_result = await http_compare.compare(self.event.data["url"], headers=headers, cookies=cookies) elif event_type == "POSTPARAM": data = {self.event.data["name"]: f"{probe}"} - if additional_params is not None: + if additional_params: data.update( self.additional_params_process( additional_params, additional_params_populate_empty From 7f496c80d1ae634c33f8e78c641e713a7bd113fb Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 20 Jul 2024 22:01:15 -0400 Subject: [PATCH 107/238] debug msg remove --- bbot/modules/internal/excavate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index cf5dacae3..416d0eb42 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -946,8 +946,6 @@ async def handle_event(self, event): custom_headers = self.scan.web_config.get("http_headers", {}) for custom_header_name, custom_header_value in custom_headers.items(): - self.hugewarning(custom_header_name) - self.hugewarning(custom_header_value) description = f"HTTP Extracted Parameter [{custom_header_name}] (Custom Header)" data = { "host": event.parsed_url.hostname, From c73bf9b0d79fc09c1156d57f877797b6e489d00a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 20 Jul 2024 23:37:39 -0400 Subject: [PATCH 108/238] small bug fix --- bbot/modules/lightfuzz_submodules/sqli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 6ed4ce376..c6a2f17ad 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -30,7 +30,7 @@ async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) if "original_value" in self.event.data and self.event.data["original_value"] is not None: - probe_value = urllib.parse.quote(self.event.data["original_value"], safe="") + probe_value = urllib.parse.quote(str(self.event.data["original_value"]), safe="") else: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) From 1a90154588e79dca00cec4467a66e044baad260a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 11:46:13 -0400 Subject: [PATCH 109/238] fixing add_get_params, allowing malicious querystrings through --- bbot/core/helpers/url.py | 28 ++++++++++--- bbot/modules/internal/excavate.py | 9 +++-- bbot/modules/lightfuzz_submodules/base.py | 21 +++++----- bbot/modules/lightfuzz_submodules/crypto.py | 45 ++++++++++++++------- bbot/scanner/preset/args.py | 1 - bbot/scanner/scanner.py | 1 - 6 files changed, 70 insertions(+), 35 deletions(-) diff --git a/bbot/core/helpers/url.py b/bbot/core/helpers/url.py index 5482e54c5..522e40b4c 100644 --- a/bbot/core/helpers/url.py +++ b/bbot/core/helpers/url.py @@ -1,7 +1,7 @@ import uuid import logging from contextlib import suppress -from urllib.parse import urlparse, parse_qs, urlencode, ParseResult +from urllib.parse import urlparse, parse_qsl, urlencode, ParseResult from .regexes import double_slash_regex @@ -32,7 +32,10 @@ def parse_url(url): return urlparse(url) -def add_get_params(url, params): +def add_get_params(url, params, encode=True): + def _no_encode_quote(s, safe="/", encoding=None, errors=None): + return s + """ Add or update query parameters to the given URL. @@ -53,10 +56,23 @@ def add_get_params(url, params): >>> add_get_params('https://www.evilcorp.com?foo=1', {'foo': 2}) ParseResult(scheme='https', netloc='www.evilcorp.com', path='', params='', query='foo=2', fragment='') """ - parsed = parse_url(url) - old_params = dict(parse_qs(parsed.query)) - old_params.update(params) - return parsed._replace(query=urlencode(old_params, doseq=True)) + parsed = urlparse(url) + query_params = parsed.query.split("&") + + existing_params = {} + for param in query_params: + if "=" in param: + k, v = param.split("=", 1) + existing_params[k] = v + + existing_params.update(params) + + if encode: + new_query = urlencode(existing_params, doseq=True) + else: + new_query = urlencode(existing_params, doseq=True, quote_via=_no_encode_quote) + + return parsed._replace(query=new_query) def get_get_params(url): diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 416d0eb42..9ea59f4ff 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -940,10 +940,11 @@ async def handle_event(self, event): "description": description, "additional_params": _exclude_key(custom_cookies, custom_cookie_name), } - context = f"Excavate saw a custom cookie set [{custom_cookie_name}], and emitted a WEB_PARAMETER for it" + context = ( + f"Excavate saw a custom cookie set [{custom_cookie_name}], and emitted a WEB_PARAMETER for it" + ) await self.emit_event(data, "WEB_PARAMETER", event, context=context) - custom_headers = self.scan.web_config.get("http_headers", {}) for custom_header_name, custom_header_value in custom_headers.items(): description = f"HTTP Extracted Parameter [{custom_header_name}] (Custom Header)" @@ -956,7 +957,9 @@ async def handle_event(self, event): "description": description, "additional_params": _exclude_key(custom_headers, custom_header_name), } - context = f"Excavate saw a custom header set [{custom_header_name}], and emitted a WEB_PARAMETER for it" + context = ( + f"Excavate saw a custom header set [{custom_header_name}], and emitted a WEB_PARAMETER for it" + ) await self.emit_event(data, "WEB_PARAMETER", event, context=context) data = event.data diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 938869770..58222585d 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -1,5 +1,6 @@ import copy + class BaseLightfuzz: def __init__(self, lightfuzz, event): self.lightfuzz = lightfuzz @@ -19,7 +20,7 @@ def additional_params_process(self, additional_params, additional_params_populat async def send_probe(self, probe): getparams = {self.event.data["name"]: probe} - url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams).geturl() + url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl() self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}") r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10) if r: @@ -37,7 +38,9 @@ def compare_baseline( if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" if self.event.data["additional_params"] is not None: - baseline_url = self.lightfuzz.helpers.add_get_params(baseline_url,self.event.data["additional_params"]).geturl() + baseline_url = self.lightfuzz.helpers.add_get_params( + baseline_url, self.event.data["additional_params"], encode=False + ).geturl() http_compare = self.lightfuzz.helpers.http_compare( baseline_url, cookies=cookies, include_cache_buster=False ) @@ -71,7 +74,7 @@ async def compare_probe( probe, cookies, additional_params_populate_empty=False, - additional_params_override = {}, + additional_params_override={}, speculative_mode="GETPARAM", ): @@ -86,7 +89,7 @@ async def compare_probe( if event_type == "GETPARAM": probe_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" if additional_params: - probe_url = self.lightfuzz.helpers.add_get_params(probe_url,additional_params).geturl() + probe_url = self.lightfuzz.helpers.add_get_params(probe_url, additional_params, encode=False).geturl() compare_result = await http_compare.compare(probe_url, cookies=cookies) elif event_type == "COOKIE": cookies_probe = {self.event.data["name"]: probe} @@ -97,11 +100,7 @@ async def compare_probe( elif event_type == "POSTPARAM": data = {self.event.data["name"]: f"{probe}"} if additional_params: - data.update( - self.additional_params_process( - additional_params, additional_params_populate_empty - ) - ) + data.update(self.additional_params_process(additional_params, additional_params_populate_empty)) compare_result = await http_compare.compare( self.event.data["url"], method="POST", data=data, cookies=cookies ) @@ -124,7 +123,9 @@ async def standard_probe( if event_type == "GETPARAM": url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}" if self.event.data["additional_params"] is not None: - url = self.lightfuzz.helpers.add_get_params(url,self.event.data["additional_params"]).geturl() + url = self.lightfuzz.helpers.add_get_params( + url, self.event.data["additional_params"], encode=False + ).geturl() else: url = self.event.data["url"] if event_type == "COOKIE": diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 3c395b38f..9d1a1e532 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -127,7 +127,6 @@ def possible_block_sizes(ciphertext_length): possible_sizes.append(block_size) return possible_sizes - async def padding_oracle_execute(self, original_data, encoding, cookies, possible_first_byte=False): possible_block_sizes = self.possible_block_sizes(len(original_data)) @@ -142,13 +141,18 @@ async def padding_oracle_execute(self, original_data, encoding, cookies, possibl baseline_byte = b"\x00" starting_pos = 1 - baseline = self.compare_baseline(self.event.data["type"], ivblock[:-1] + baseline_byte + datablock, cookies) + baseline = self.compare_baseline( + self.event.data["type"], ivblock[:-1] + baseline_byte + datablock, cookies + ) differ_count = 0 for i in range(starting_pos, starting_pos + 254): byte = bytes([i]) oracle_probe = await self.compare_probe( - baseline, self.event.data["type"], self.format_agnostic_encode(ivblock[:-1] + byte + datablock, encoding), cookies + baseline, + self.event.data["type"], + self.format_agnostic_encode(ivblock[:-1] + byte + datablock, encoding), + cookies, ) if oracle_probe[0] == False and "body" in oracle_probe[1]: @@ -171,7 +175,9 @@ async def padding_oracle(self, probe_value, cookies): padding_oracle_result, block_size = await self.padding_oracle_execute(data, encoding, cookies) if padding_oracle_result == None: self.lightfuzz.debug("ended up in possible_first_byte situation - retrying with different first byte") - padding_oracle_result = await self.padding_oracle_execute(data, encoding, cookies, possible_first_byte=True) + padding_oracle_result = await self.padding_oracle_execute( + data, encoding, cookies, possible_first_byte=True + ) if padding_oracle_result == True: context = f"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]" @@ -212,7 +218,7 @@ def identify_hash_function(hash_bytes): 20: hashlib.sha1, 32: hashlib.sha256, 48: hashlib.sha384, - 64: hashlib.sha512 + 64: hashlib.sha512, } if hash_length in hash_functions: @@ -235,7 +241,9 @@ async def fuzz(self): truncate_probe_value = self.modify_string(probe_value, action="truncate") mutate_probe_value = self.modify_string(probe_value, action="mutate") except ValueError as e: - self.lightfuzz.debug(f"Encountered error modifying value for parameter {self.event.data['name']}: {e} , aborting") + self.lightfuzz.debug( + f"Encountered error modifying value for parameter {self.event.data['name']}: {e} , aborting" + ) return # Basic crypanalysis @@ -252,14 +260,17 @@ async def fuzz(self): truncate_probe = await self.compare_probe(http_compare, self.event.data["type"], truncate_probe_value, cookies) mutate_probe = await self.compare_probe(http_compare, self.event.data["type"], mutate_probe_value, cookies) - confirmed_techniques = [] - if (mutate_probe[0] == False and "body" in mutate_probe[1]): - if (http_compare.compare_body(mutate_probe[3].text, arbitrary_probe[3].text) == False) or mutate_probe[3].text == "": + if mutate_probe[0] == False and "body" in mutate_probe[1]: + if (http_compare.compare_body(mutate_probe[3].text, arbitrary_probe[3].text) == False) or mutate_probe[ + 3 + ].text == "": confirmed_techniques.append("Single-byte Mutation") if mutate_probe[0] == False and "body" in mutate_probe[1]: - if (http_compare.compare_body(truncate_probe[3].text, arbitrary_probe[3].text) == False) or truncate_probe[3].text == "": + if (http_compare.compare_body(truncate_probe[3].text, arbitrary_probe[3].text) == False) or truncate_probe[ + 3 + ].text == "": confirmed_techniques.append("Data Truncation") if confirmed_techniques: @@ -277,16 +288,16 @@ async def fuzz(self): {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text} ) - if confirmed_techniques: # Padding Oracle Test if possible_block_cipher: - self.lightfuzz.debug("Attempting padding oracle exploit since it looks like a block cipher and we have confirmed crypto") + self.lightfuzz.debug( + "Attempting padding oracle exploit since it looks like a block cipher and we have confirmed crypto" + ) await self.padding_oracle(probe_value, cookies) - # Hash identification / Potential Length extension attack data, encoding = CryptoLightfuzz.format_agnostic_decode(probe_value) @@ -297,7 +308,13 @@ async def fuzz(self): # this would indicate they are being used to generate the hash if hash_function and self.event.data["additional_params"]: for additional_param_name, additional_param_value in self.event.data["additional_params"].items(): - additional_param_probe = await self.compare_probe(http_compare, self.event.data["type"], probe_value, cookies, additional_params_override={additional_param_name: additional_param_value + "A"}) + additional_param_probe = await self.compare_probe( + http_compare, + self.event.data["type"], + probe_value, + cookies, + additional_params_override={additional_param_name: additional_param_value + "A"}, + ) # the additional parameter affects the potential hash parameter (suggesting its calculated in the hash) if additional_param_probe[0] == False and (additional_param_probe[1] == mutate_probe[1]): context = f"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) that is a likely a hash, which is connected to another parameter {additional_param_name})" diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index bde9a2f8b..28af73b9a 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -382,7 +382,6 @@ def sanitize_args(self): custom_cookies_dict[k] = v self.parsed.custom_cookies = custom_cookies_dict - def validate(self): # validate config options sentinel = object() diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 50618b12a..585e9f441 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -207,7 +207,6 @@ def __init__( "You have enabled custom HTTP cookies. These will be attached to all in-scope requests and all requests made by httpx." ) - # url file extensions self.url_extension_blacklist = set(e.lower() for e in self.config.get("url_extension_blacklist", [])) self.url_extension_httpx_only = set(e.lower() for e in self.config.get("url_extension_httpx_only", [])) From d940f9cc0abb6abf4177f3e310bc53400614154a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 11:56:35 -0400 Subject: [PATCH 110/238] bug fixes --- bbot/core/helpers/url.py | 2 +- bbot/test/test_step_2/module_tests/test_module_lightfuzz.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bbot/core/helpers/url.py b/bbot/core/helpers/url.py index 522e40b4c..91c7d8a01 100644 --- a/bbot/core/helpers/url.py +++ b/bbot/core/helpers/url.py @@ -1,7 +1,7 @@ import uuid import logging from contextlib import suppress -from urllib.parse import urlparse, parse_qsl, urlencode, ParseResult +from urllib.parse import urlparse, parse_qs, urlencode, ParseResult from .regexes import double_slash_regex diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 58fd80d79..346946e87 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -818,10 +818,7 @@ def check(self, module_test, events): web_parameter_emitted = False cmdi_interacttsh_finding_emitted = False - print("@@@@") for e in events: - print(e) - print(e.type) if e.type == "WEB_PARAMETER": print(e.data["description"]) if "HTTP Extracted Parameter [search]" in e.data["description"]: From bdf8683c0afb2eff36d4d7ba4c124a92182e90b0 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 13:20:13 -0400 Subject: [PATCH 111/238] sometimes no additional params --- bbot/modules/lightfuzz_submodules/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 58222585d..9c0ad97ad 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -37,7 +37,7 @@ def compare_baseline( if event_type == "GETPARAM": baseline_url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" - if self.event.data["additional_params"] is not None: + if "additional_params" in self.event.data.keys() and self.event.data["additional_params"] is not None: baseline_url = self.lightfuzz.helpers.add_get_params( baseline_url, self.event.data["additional_params"], encode=False ).geturl() From a82e7fca3eca371232dea307e0721100af9f8ea2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 13:59:03 -0400 Subject: [PATCH 112/238] bug fix. --- bbot/modules/lightfuzz_submodules/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 9c0ad97ad..1db601cca 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -122,7 +122,7 @@ async def standard_probe( method = "GET" if event_type == "GETPARAM": url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}" - if self.event.data["additional_params"] is not None: + if "additional_params" in self.event.data.keys() and self.event.data["additional_params"] is not None: url = self.lightfuzz.helpers.add_get_params( url, self.event.data["additional_params"], encode=False ).geturl() From 2d594634f66adf892032c234feb5b2ca529caa9f Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 22:56:06 -0400 Subject: [PATCH 113/238] another lil bug --- bbot/modules/lightfuzz_submodules/cmdi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index a5561841c..2765ae34a 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -10,7 +10,7 @@ async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) if ( - "original_value" in self.event.data + "original_value" in self.event.data.keys() and self.event.data["original_value"] is not None and len(self.event.data["original_value"]) != 0 ): From 84b9967f912adb9d3f8978fd49ef1c5198dd25ba Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 23:02:20 -0400 Subject: [PATCH 114/238] better fix --- bbot/modules/lightfuzz_submodules/cmdi.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index 2765ae34a..dc69c3278 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -9,12 +9,9 @@ class CmdILightFuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - if ( - "original_value" in self.event.data.keys() - and self.event.data["original_value"] is not None - and len(self.event.data["original_value"]) != 0 - ): - probe_value = self.event.data["original_value"] + original_value = self.event.data.get("original_value", None) + if original_value is not None and len(original_value) != 0: + probe_value = original_value else: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) From 6cb5c1978d8b9bca8d77d256f04145aa5a0c440e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 23:11:17 -0400 Subject: [PATCH 115/238] 3rd try :| --- bbot/modules/lightfuzz_submodules/cmdi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index dc69c3278..fb421da6e 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -9,7 +9,7 @@ class CmdILightFuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - original_value = self.event.data.get("original_value", None) + original_value = str(self.event.data.get("original_value", "")) if original_value is not None and len(original_value) != 0: probe_value = original_value else: From c8329c7854dfc37b60b3acacea7bc6b3e2f8ec8c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 23:22:48 -0400 Subject: [PATCH 116/238] original value fix --- bbot/modules/lightfuzz_submodules/crypto.py | 7 ++----- bbot/modules/lightfuzz_submodules/path.py | 8 +++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 9d1a1e532..d0b5c3b27 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -226,11 +226,8 @@ def identify_hash_function(hash_bytes): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - if ( - "original_value" in self.event.data - and self.event.data["original_value"] is not None - and self.event.data["original_value"] != "1" - ): + original_value = self.event.data.get("original_value", None) + if original_value is not None and original_value != "1": probe_value = self.event.data["original_value"] else: self.lightfuzz.debug( diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 5de44082b..4be07b39e 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -9,11 +9,9 @@ class PathTraversalLightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - if ( - "original_value" in self.event.data - and self.event.data["original_value"] is not None - and self.event.data["original_value"] != "1" - ): + + original_value = self.event.data.get("original_value", None) + if original_value is not None and original_value != "1": probe_value = self.event.data["original_value"] else: self.lightfuzz.debug( From bf4b5be9c9d6dcc3d5e750e92ce0e537b6d9db62 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 23:28:58 -0400 Subject: [PATCH 117/238] one more --- bbot/modules/lightfuzz_submodules/path.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 4be07b39e..57e11d4ad 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -10,9 +10,9 @@ class PathTraversalLightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - original_value = self.event.data.get("original_value", None) + original_value = str(self.event.data.get("original_value", "")) if original_value is not None and original_value != "1": - probe_value = self.event.data["original_value"] + probe_value = original_value else: self.lightfuzz.debug( f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" From 54b2fcb572697be92577f26cb3f9fff70c667de3 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 21 Jul 2024 23:35:24 -0400 Subject: [PATCH 118/238] handle non-str strings --- bbot/modules/lightfuzz_submodules/crypto.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index d0b5c3b27..f93ddacc1 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -75,6 +75,9 @@ def format_agnostic_encode(data, encoding, urlencode=True): @staticmethod def modify_string(input_string, action="truncate", position=None, extension_length=1): + if not isinstance(input_string, str): + input_string = str(input_string) + data, encoding = CryptoLightfuzz.format_agnostic_decode(input_string) if encoding != "base64" and encoding != "hex": raise ValueError("Input must be either hex or base64 encoded") From 23c414fb90a9067557209e077d470e43c36bb0ff Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 24 Jul 2024 08:48:12 -0400 Subject: [PATCH 119/238] tweaks to form extraction --- bbot/modules/internal/excavate.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 9ea59f4ff..0b7ec13fa 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -403,7 +403,7 @@ class PostJquery(GetJquery): class HtmlTags(ParameterExtractorRule): name = "HTML Tags" - discovery_regex = r'/<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/ nocase' + discovery_regex = r'/<[^>]+(href|src|action)=["\'][^"\']*["\'][^>]*>/ nocase' extraction_regex = bbot_regexes.tag_attribute_regex output_type = "GETPARAM" @@ -446,6 +446,10 @@ def extract(self): form_parameters, parameter_name ) + class GenericForm(GetForm): + name = "Generic Form" + discovery_regex = r'/form[^>]*>.*<\/form>/s nocase' + class PostForm(GetForm): name = "POST Form" discovery_regex = r'/]*\bmethod=["\']?post["\']?[^>]*>.*<\/form>/s nocase' @@ -676,7 +680,7 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte class URLExtractor(ExcavateRule): yara_rules = { "url_full": r'rule url_full { meta: tags = "spider-danger" description = "contains full URL" strings: $url_full = /https?:\/\/([\w\.-]+)([:\/\w\.-]*)/ condition: $url_full }', - "url_attr": r'rule url_attr { meta: tags = "spider-danger" description = "contains tag with src or href attribute" strings: $url_attr = /<[^>]+(href|src)=["\'][^"\']*["\'][^>]*>/ condition: $url_attr }', + "url_attr": r'rule url_attr { meta: tags = "spider-danger" description = "contains tag with src or href attribute" strings: $url_attr = /<[^>]+(href|src|action)=["\'][^"\']*["\'][^>]*>/ condition: $url_attr }', } full_url_regex = re.compile(r"(https?)://((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)") full_url_regex_strict = re.compile(r"^(https?):\/\/([\w.-]+)(?::\d{1,5})?(\/[\w\/\.-]*)?(\?[^\s]+)?$") From 197b5796eb2775a424d4c57de2fe69d2111f40e3 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 8 Aug 2024 23:26:43 -0400 Subject: [PATCH 120/238] fixing unreachable host issues --- bbot/core/helpers/diff.py | 4 ++-- bbot/core/helpers/helper.py | 2 +- bbot/modules/internal/excavate.py | 2 +- bbot/modules/lightfuzz.py | 17 ++++++++++++----- bbot/modules/lightfuzz_submodules/base.py | 1 - bbot/modules/lightfuzz_submodules/path.py | 5 +---- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 212a73caa..138d73ac0 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -19,7 +19,7 @@ def __init__( include_cache_buster=True, headers=None, cookies=None, - timeout=15, + timeout=10, ): self.parent_helper = parent_helper self.baseline_url = baseline_url @@ -30,7 +30,7 @@ def __init__( self._baselined = False self.headers = headers self.cookies = cookies - self.timeout = 15 + self.timeout = 10 @staticmethod def merge_dictionaries(headers1, headers2): diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 77aa22566..2e2df6f61 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -122,7 +122,7 @@ def http_compare( cookies=None, method="GET", data=None, - timeout=15, + timeout=10, ): return HttpCompare( url, diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 0b7ec13fa..e4a029fe5 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -448,7 +448,7 @@ def extract(self): class GenericForm(GetForm): name = "Generic Form" - discovery_regex = r'/form[^>]*>.*<\/form>/s nocase' + discovery_regex = r"/form[^>]*>.*<\/form>/s nocase" class PostForm(GetForm): name = "POST Form" diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 37f2b06e6..355f13532 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -64,7 +64,7 @@ async def setup(self): for m in self.enabled_submodules: if m not in self.submodules: - self.hugewarning(f"Invalid Lightfuzz submodule ({m}) specified in enabeld_modules") + self.hugewarning(f"Invalid Lightfuzz submodule ({m}) specified in enabled_modules") return False for submodule, submodule_dict in self.submodules.items(): @@ -174,10 +174,17 @@ async def handle_event(self, event): await self.emit_event(data, "WEB_PARAMETER", event) elif event.type == "WEB_PARAMETER": - for submodule, submodule_dict in self.submodules.items(): - if getattr(self, submodule): - self.debug(f"Starting {submodule_dict['description']} fuzz()") - await self.run_submodule(submodule_dict["module"], event) + + # check connectivity to url + connectivity_test = await self.helpers.request(event.data["url"], timeout=10) + + if connectivity_test: + for submodule, submodule_dict in self.submodules.items(): + if getattr(self, submodule): + self.debug(f"Starting {submodule_dict['description']} fuzz()") + await self.run_submodule(submodule_dict["module"], event) + else: + self.debug(f'WEB_PARAMETER URL {event.data["url"]} failed connectivity test, aborting') async def cleanup(self): if self.interactsh_instance: diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 1db601cca..7ce5b0f34 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -77,7 +77,6 @@ async def compare_probe( additional_params_override={}, speculative_mode="GETPARAM", ): - additional_params = copy.deepcopy(self.event.data.get("additional_params", {})) if additional_params_override: for k, v in additional_params_override.items(): diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 57e11d4ad..cae82b2da 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -12,7 +12,7 @@ async def fuzz(self): original_value = str(self.event.data.get("original_value", "")) if original_value is not None and original_value != "1": - probe_value = original_value + probe_value = urllib.parse.quote(original_value) else: self.lightfuzz.debug( f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" @@ -54,9 +54,7 @@ async def fuzz(self): confirmations = 0 while iterations > 0: try: - http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) - singledot_probe = await self.compare_probe( http_compare, self.event.data["type"], payloads["singledot_payload"], cookies ) @@ -70,7 +68,6 @@ async def fuzz(self): and doubledot_probe[3] != None and doubledot_probe[1] != ["header"] ): - confirmations += 1 self.lightfuzz.verbose( f"Got possible Path Traversal detection: [{str(confirmations)}] Confirmations" From 8a6a4caf0b737d5d8af8ac1381dd4787031e4e99 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 10 Aug 2024 12:59:38 -0400 Subject: [PATCH 121/238] reworking parameter extraction rules, bug fixes --- bbot/core/helpers/regexes.py | 21 ++++++++++++--- bbot/modules/internal/excavate.py | 22 ++++++++++++---- .../module_tests/test_module_excavate.py | 26 ++++++++++++++++--- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 0c01ff022..b6ccf5eb0 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -118,18 +118,31 @@ input_tag_regex = re.compile( r"]+?name=[\"\']?([\.$\w]+)[\"\']?(?:[^>]*?value=[\"\']([=+\/\w]*)[\"\'])?[^>]*>" ) -jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") -jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}") +# jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") +# jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}") +# jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}") a_tag_regex = re.compile(r"]*href=[\"\']([^\"\'?>]*)\?([^&\"\'=]+)=([^&\"\'=]+)") img_tag_regex = re.compile(r"]*src=[\"\']([^\"\'?>]*)\?([^&\"\'=]+)=([^&\"\'=]+)") get_form_regex = re.compile( - r"]+(?:action=[\"']?([^\s\'\"]+)[\"\']?)?[^>]*method=[\"']?[gG][eE][tT][\"']?[^>]*>([\s\S]*?)<\/form>", + r"]*\bmethod=[\"']?[gG][eE][tT][\"']?[^>]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*>([\s\S]*?)<\/form>", + re.DOTALL, +) +get_form_regex2 = re.compile( + r"]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*\bmethod=[\"']?[gG][eE][tT][\"']?[^>]*>([\s\S]*?)<\/form>", re.DOTALL, ) post_form_regex = re.compile( - r"]+(?:action=[\"']?([^\s\'\"]+)[\"\']?)?[^>]*method=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>", + r"]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*>([\s\S]*?)<\/form>", + re.DOTALL, +) +post_form_regex2 = re.compile( + r"]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>", re.DOTALL, ) +generic_form_regex = re.compile( + r"]*\bmethod=)[^>]+(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)[^>]*>([\s\S]*?)<\/form>", re.DOTALL +) + select_tag_regex = re.compile( r"]+?name=[\"\']?(\w+)[\"\']?[^>]*>(?:\s*]*?value=[\"\'](\w*)[\"\']?[^>]*>)?" ) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index e4a029fe5..75d08ea07 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -446,9 +446,8 @@ def extract(self): form_parameters, parameter_name ) - class GenericForm(GetForm): - name = "Generic Form" - discovery_regex = r"/form[^>]*>.*<\/form>/s nocase" + class GetForm2(GetForm): + extraction_regex = bbot_regexes.get_form_regex2 class PostForm(GetForm): name = "POST Form" @@ -456,6 +455,17 @@ class PostForm(GetForm): extraction_regex = bbot_regexes.post_form_regex output_type = "POSTPARAM" + class PostForm2(PostForm): + extraction_regex = bbot_regexes.post_form_regex2 + + # underscore ensure generic forms runs last, so it doesn't cause dedupe to stop full form detection + class _GenericForm(GetForm): + name = "Generic Form" + discovery_regex = r"/]*>.*<\/form>/s nocase" + + extraction_regex = bbot_regexes.generic_form_regex + output_type = "GETPARAM" + def __init__(self, excavate): super().__init__(excavate) self.parameterExtractorCallbackDict = {} @@ -467,11 +477,13 @@ def __init__(self, excavate): regexes_component_list.append(f"${r.__name__} = {r.discovery_regex}") regexes_component = " ".join(regexes_component_list) self.yara_rules[f"parameter_extraction"] = ( - rf'rule parameter_extraction {{meta: description = "contains POST form" strings: {regexes_component} condition: any of them}}' + rf'rule parameter_extraction {{meta: description = "contains Parameter" strings: {regexes_component} condition: any of them}}' ) async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): + print("AAAAAAAAAAAAAAAAA") + print(identifier) for result in results: if identifier not in self.parameterExtractorCallbackDict.keys(): raise ExcavateError("ParameterExtractor YaraRule identified reference non-existent submodule") @@ -829,7 +841,7 @@ async def setup(self): rule_match = await self.helpers.re.search(self.yara_rule_name_regex, rule_content) if not rule_match: - self.hugewarning(f"Custom Yara formatted incorrectly: could not find rule name") + self.hugewarning(f"Custom Yara rule formatted incorrectly: could not find rule name") return False rule_name = rule_match.groups(1)[0] diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 2e00eacd6..0065568b9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -456,14 +456,21 @@ class TestExcavateParameterExtraction(TestExcavate):

Use the form below to submit a GET request:

-

+

Simple POST Form

Use the form below to submit a POST request:

-

+

+ +
+

Simple Generic Form

+

Use the form below to submit a request:

+
+ +

Links

@@ -482,15 +489,19 @@ def check(self, module_test, events): found_jquery_post = False found_form_get = False found_form_post = False + found_form_generic = False found_jquery_get_original_value = False found_jquery_post_original_value = False found_form_get_original_value = False found_form_post_original_value = False + found_form_generic_original_value = False found_htmltags_a = False found_htmltags_img = False for e in events: + if e.type == "WEB_PARAMETER": + if e.data["description"] == "HTTP Extracted Parameter [jqueryget] (GET jquery Submodule)": found_jquery_get = True if e.data["original_value"] == "value1": @@ -501,16 +512,21 @@ def check(self, module_test, events): if e.data["original_value"] == "value2": found_jquery_post_original_value = True - if e.data["description"] == "HTTP Extracted Parameter [q] (GET Form Submodule)": + if e.data["description"] == "HTTP Extracted Parameter [q1] (GET Form Submodule)": found_form_get = True if e.data["original_value"] == "flowers": found_form_get_original_value = True - if e.data["description"] == "HTTP Extracted Parameter [q] (POST Form Submodule)": + if e.data["description"] == "HTTP Extracted Parameter [q2] (POST Form Submodule)": found_form_post = True if e.data["original_value"] == "boats": found_form_post_original_value = True + if e.data["description"] == "HTTP Extracted Parameter [q3] (Generic Form Submodule)": + found_form_generic = True + if e.data["original_value"] == "candles": + found_form_generic_original_value = True + if e.data["description"] == "HTTP Extracted Parameter [age] (HTML Tags Submodule)": if e.data["original_value"] == "456": if "id" in e.data["additional_params"].keys(): @@ -525,10 +541,12 @@ def check(self, module_test, events): assert found_jquery_post, "Did not extract Jquery POST parameters" assert found_form_get, "Did not extract Form GET parameters" assert found_form_post, "Did not extract Form POST parameters" + assert found_form_generic, "Did not extract Form (Generic) parameters" assert found_jquery_get_original_value, "Did not extract Jquery GET parameter original_value" assert found_jquery_post_original_value, "Did not extract Jquery POST parameter original_value" assert found_form_get_original_value, "Did not extract Form GET parameter original_value" assert found_form_post_original_value, "Did not extract Form POST parameter original_value" + assert found_form_generic_original_value, "Did not extract Form (Generic) parameter original_value" assert found_htmltags_a, "Did not extract parameter(s) from a-tag" assert found_htmltags_img, "Did not extract parameter(s) from img-tag" From 67b69d19193d68a73a5ca680675109b9728fbd40 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 10 Aug 2024 13:03:17 -0400 Subject: [PATCH 122/238] removing debug --- bbot/modules/internal/excavate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 75d08ea07..2ef527432 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -482,8 +482,6 @@ def __init__(self, excavate): async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): - print("AAAAAAAAAAAAAAAAA") - print(identifier) for result in results: if identifier not in self.parameterExtractorCallbackDict.keys(): raise ExcavateError("ParameterExtractor YaraRule identified reference non-existent submodule") From 1b00b0609b36069719e2e5d6a80de6787c3a1a8d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 12 Aug 2024 14:04:43 -0400 Subject: [PATCH 123/238] adding lightfuzz serialization submodule --- bbot/modules/internal/excavate.py | 12 +- bbot/modules/lightfuzz.py | 6 +- bbot/modules/lightfuzz_submodules/cmdi.py | 2 +- bbot/modules/lightfuzz_submodules/serial.py | 63 ++++++ .../module_tests/test_module_lightfuzz.py | 188 ++++++++++++++++++ 5 files changed, 264 insertions(+), 7 deletions(-) create mode 100644 bbot/modules/lightfuzz_submodules/serial.py diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 8413ddb08..44e3e94b5 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -7,7 +7,7 @@ from bbot.errors import ExcavateError import bbot.core.helpers.regexes as bbot_regexes from bbot.modules.internal.base import BaseInternalModule -from urllib.parse import urlparse, urljoin, parse_qs, urlunparse +from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, quote def find_subclasses(obj, base_class): @@ -391,7 +391,7 @@ def extract(self): for action, extracted_parameters in extracted_results: extracted_parameters_dict = self.convert_to_dict(extracted_parameters) for parameter_name, original_value in extracted_parameters_dict.items(): - yield self.output_type, parameter_name, original_value, action, _exclude_key( + yield self.output_type, parameter_name, original_value.strip(), action, _exclude_key( extracted_parameters_dict, parameter_name ) @@ -416,7 +416,7 @@ def extract(self): k: v[0] if isinstance(v, list) and len(v) == 1 else v for k, v in query_strings.items() } for parameter_name, original_value in query_strings_dict.items(): - yield self.output_type, parameter_name, original_value, url, _exclude_key( + yield self.output_type, parameter_name, original_value.strip(), url, _exclude_key( query_strings_dict, parameter_name ) @@ -434,12 +434,16 @@ class GetForm(ParameterExtractorRule): def extract(self): forms = self.extraction_regex.findall(str(self.result)) for form_action, form_content in forms: + + if form_action.startswith("./"): + form_action = form_action.lstrip(".") + form_parameters = {} for form_content_regex in self.form_content_regexes: input_tags = form_content_regex.findall(form_content) for parameter_name, original_value in input_tags: - form_parameters[parameter_name] = original_value + form_parameters[parameter_name] = original_value.strip() for parameter_name, original_value in form_parameters.items(): yield self.output_type, parameter_name, original_value, form_action, _exclude_key( diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 355f13532..1d0737218 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -8,12 +8,13 @@ from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote from bbot.errors import InteractshError, HttpCompareError -from .lightfuzz_submodules.cmdi import CmdILightFuzz +from .lightfuzz_submodules.cmdi import CmdILightfuzz from .lightfuzz_submodules.crypto import CryptoLightfuzz from .lightfuzz_submodules.path import PathTraversalLightfuzz from .lightfuzz_submodules.sqli import SQLiLightfuzz from .lightfuzz_submodules.ssti import SSTILightfuzz from .lightfuzz_submodules.xss import XSSLightfuzz +from .lightfuzz_submodules.serial import SerialLightfuzz class lightfuzz(BaseModule): @@ -23,11 +24,12 @@ class lightfuzz(BaseModule): submodules = { "sqli": {"description": "SQL Injection", "module": SQLiLightfuzz}, - "cmdi": {"description": "Command Injection", "module": CmdILightFuzz}, + "cmdi": {"description": "Command Injection", "module": CmdILightfuzz}, "xss": {"description": "Cross-site Scripting", "module": XSSLightfuzz}, "path": {"description": "Path Traversal", "module": PathTraversalLightfuzz}, "ssti": {"description": "Server-side Template Injection", "module": SSTILightfuzz}, "crypto": {"description": "Cryptography Probe", "module": CryptoLightfuzz}, + "serial": {"description": "Unsafe Deserialization Probe", "module": SerialLightfuzz}, } options = {"force_common_headers": False, "enabled_submodules": []} diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index fb421da6e..671574c95 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -4,7 +4,7 @@ import urllib.parse -class CmdILightFuzz(BaseLightfuzz): +class CmdILightfuzz(BaseLightfuzz): async def fuzz(self): diff --git a/bbot/modules/lightfuzz_submodules/serial.py b/bbot/modules/lightfuzz_submodules/serial.py new file mode 100644 index 000000000..395ff8ebb --- /dev/null +++ b/bbot/modules/lightfuzz_submodules/serial.py @@ -0,0 +1,63 @@ +from .base import BaseLightfuzz +from bbot.errors import HttpCompareError + + +class SerialLightfuzz(BaseLightfuzz): + async def fuzz(self): + + cookies = self.event.data.get("assigned_cookies", {}) + control_payload = "DEADBEEFCAFEBABE1234567890ABCDEF" + serialization_payloads = { + "php_base64": "YTowOnt9", + "php_raw": "a:0:{}", + "java_hex": "ACED00057372000E6A6176612E6C616E672E426F6F6C65616ECD207EC0D59CF6EE02000157000576616C7565787000", + "java_base64": "rO0ABXNyABFqYXZhLmxhbmcuQm9vbGVhbs0gcoDVnPruAgABWgAFdmFsdWV4cAA=", + "java_base64_string_error": "rO0ABXQABHRlc3Q=", + "java_base64_OptionalDataException": "rO0ABXcEAAAAAAEAAAABc3IAEGphdmEudXRpbC5IYXNoTWFwAAAAAAAAAAECAAJMAARrZXkxYgABAAAAAAAAAAJ4cHcBAAAAB3QABHRlc3Q=", + "java_hex_OptionalDataException": "ACED0005737200106A6176612E7574696C2E486173684D617000000000000000012000014C00046B6579317A00010000000000000278707000000774000474657374", + "dotnet_hex": "0001000000ffffffff01000000000000000601000000076775737461766f0b", + "dotnet_base64": "AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==", + "ruby_base64": "BAh7BjoKbE1FAAVJsg==", + } + + serialization_errors = [ + "invalid user", + "cannot cast java.lang.string", + "dump format error", + "java.io.optionaldataexception", + ] + + original_value = self.event.data.get("original_value", None).strip() + if not (original_value == None or original_value == ""): + self.lightfuzz.debug( + f"The Serialization Submodule only operates when there if no original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" + ) + return + + http_compare = self.compare_baseline(self.event.data["type"], control_payload, cookies) + for type, payload in serialization_payloads.items(): + try: + serialization_probe = await self.compare_probe(http_compare, self.event.data["type"], payload, cookies) + if serialization_probe[0] == False and serialization_probe[1] != ["header"]: + if serialization_probe[3].status_code == 200 and "code" in serialization_probe[1]: + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Unsafe Deserialization. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Technique: [Error Resolution] Serialization Payload: [{type}]", + } + ) + elif serialization_probe[3].status_code == 500 or ( + serialization_probe[3].status_code == 200 and serialization_probe[1] == ["body"] + ): + for serialization_error in serialization_errors: + if serialization_error in serialization_probe[3].text.lower(): + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Unsafe Deserialization. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Technique: [Differential Error Analysis] Error-String: [{serialization_error}] Payload: [{type}]", + } + ) + break + except HttpCompareError as e: + self.lightfuzz.debug(e) + continue diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 346946e87..1b42e2763 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -690,6 +690,194 @@ def check(self, module_test, events): assert sqldelay_finding_emitted, "SQLi Delay FINDING not emitted" +# Serialization Module (Error Resolution) +class Test_Lightfuzz_serial_errorresolution(ModuleTestBase): + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "lightfuzz", "excavate"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "enabled_submodules": ["serial"], + } + }, + } + + async def setup_after_prep(self, module_test): + + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def request_handler(self, request): + + dotnet_serial_error = """ + + Description: An unhandled exception occurred during the execution of the current web request. Please review the stack trace for more information about the error and where it originated in the code. + +

+ + Exception Details: System.Runtime.Serialization.SerializationException: End of Stream encountered before parsing was completed.

+ + """ + + dotnet_serial_error_resolved = ( + "Deserialization successful! Object type: System.String" + ) + + dotnet_serial_html = """ + + + + Deserialization RCE Example + + +
+
+ +
+ +
+ + + +
+
+

Deserialization Test

+ Enter serialized data:
+

+

+
+
+ + + + + """ + + post_params = request.form + + if "TextBox1" not in post_params.keys(): + return Response(dotnet_serial_html, status=200) + + else: + if post_params["__VIEWSTATE"] != "/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP": + return Response(dotnet_serial_error, status=500) + if post_params["TextBox1"] == "AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==": + + return Response(dotnet_serial_error_resolved, status=200) + else: + return Response(dotnet_serial_error, status=500) + + def check(self, module_test, events): + + excavate_extracted_form_parameter = False + excavate_extracted_form_parameter_details = False + lightfuzz_serial_detect_errorresolution = False + lightfuzz_serial_detect_errordifferential = False + + for e in events: + if e.type == "WEB_PARAMETER": + if e.data["name"] == "TextBox1": + excavate_extracted_form_parameter = True + if ( + e.data["url"] == "http://127.0.0.1:8888/deser.aspx" + and e.data["host"] == "127.0.0.1" + and e.data["additional_params"] + == { + "__VIEWSTATE": "/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP", + "__VIEWSTATEGENERATOR": "AD6F025C", + "__EVENTVALIDATION": "/wEdAANdCjkiIFhjCB8ta8aO/EhuESCFkFW/RuhzY1oLb/NUVM34O/GfAV4V4n0wgFZHr3czZjft8VgObR/WUivai7w4kfR1wg==", + "Button1": "Submit", + } + ): + excavate_extracted_form_parameter_details = True + if e.type == "FINDING": + if ( + e.data["description"] + == "POSSIBLE Unsafe Deserialization. Parameter: [TextBox1] Parameter Type: [POSTPARAM] Technique: [Error Resolution] Serialization Payload: [dotnet_base64]" + ): + lightfuzz_serial_detect_errorresolution = True + + assert excavate_extracted_form_parameter, "WEB_PARAMETER for POST form was not emitted" + assert excavate_extracted_form_parameter_details, "WEB_PARAMETER for POST form did not have correct data" + assert ( + lightfuzz_serial_detect_errorresolution + ), "Lightfuzz Serial module failed to detect ASP.NET error resolution based deserialization" + + +# Serialization Module (Error Differential) +class Test_Lightfuzz_serial_errordifferential(Test_Lightfuzz_serial_errorresolution): + + def request_handler(self, request): + + java_serial_error = """ + +

Internal Server Error

+

java.io.StreamCorruptedException: invalid stream header: 0C400304

+ + """ + + java_serial_error_keyword = """ + +

Internal Server Error

+

java.lang.ClassCastException: Cannot cast java.lang.String to lab.actions.common.serializable.AccessTokenUser

+ + """ + + java_serial_html = """ + + + + Deserialization RCE Example + + + Please log in to continue. + + + """ + + cookies = request.cookies + + if "session" not in cookies.keys(): + + response = Response(java_serial_html, status=200) + response.set_cookie("session", value="", max_age=3600, httponly=True) + return response + + else: + if cookies["session"] == "rO0ABXQABHRlc3Q=": + + return Response(java_serial_error_keyword, status=500) + else: + + return Response(java_serial_error, status=500) + + def check(self, module_test, events): + excavate_extracted_cookie_parameter = False + lightfuzz_serial_detect_errordifferential = False + + for e in events: + if e.type == "WEB_PARAMETER": + + if e.data["description"] == "Set-Cookie Assigned Cookie [session]" and e.data["type"] == "COOKIE": + excavate_extracted_cookie_parameter = True + + if e.type == "FINDING": + print(e.data["description"]) + + if ( + e.data["description"] + == "POSSIBLE Unsafe Deserialization. Parameter: [session] Parameter Type: [COOKIE] Technique: [Differential Error Analysis] Error-String: [cannot cast java.lang.string] Payload: [java_base64_string_error]" + ): + lightfuzz_serial_detect_errordifferential = True + + assert excavate_extracted_cookie_parameter, "WEB_PARAMETER for cookie was not emitted" + assert ( + lightfuzz_serial_detect_errordifferential + ), "Lightfuzz Serial module failed to detect Java error differential based deserialization" + + # CMDi echo canary class Test_Lightfuzz_cmdi(ModuleTestBase): targets = ["http://127.0.0.1:8888"] From bd4b7601be9d3aec35903012f4376a45d3c38ce2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Mon, 12 Aug 2024 15:30:23 -0400 Subject: [PATCH 124/238] ruby serialization signature --- bbot/modules/internal/excavate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 44e3e94b5..6380a2a87 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -611,6 +611,7 @@ class SerializationExtractor(ExcavateRule): regexes = { "Java": re.compile(r"[^a-zA-Z0-9\/+]rO0[a-zA-Z0-9+\/]+={0,2}"), + "Ruby": re.compile(r"[^a-zA-Z0-9\/+]BAh[a-zA-Z0-9+\/]+={0,2}"), "DOTNET": re.compile(r"[^a-zA-Z0-9\/+]AAEAAAD\/\/[a-zA-Z0-9\/+]+={0,2}"), "PHP_Array": re.compile(r"[^a-zA-Z0-9\/+]YTo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), "PHP_String": re.compile(r"[^a-zA-Z0-9\/+]czo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), From 658f0e00ce604e93bc33d6f9e35c17568d94da75 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 13 Aug 2024 22:47:14 -0400 Subject: [PATCH 125/238] unnecessary strip --- bbot/modules/lightfuzz_submodules/serial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/serial.py b/bbot/modules/lightfuzz_submodules/serial.py index 395ff8ebb..7ab6cfc5d 100644 --- a/bbot/modules/lightfuzz_submodules/serial.py +++ b/bbot/modules/lightfuzz_submodules/serial.py @@ -27,7 +27,7 @@ async def fuzz(self): "java.io.optionaldataexception", ] - original_value = self.event.data.get("original_value", None).strip() + original_value = self.event.data.get("original_value", None) if not (original_value == None or original_value == ""): self.lightfuzz.debug( f"The Serialization Submodule only operates when there if no original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" From 52937bbba3a0fbac58f2e39720c829f4cbac0b8b Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 14 Aug 2024 16:09:57 -0400 Subject: [PATCH 126/238] tweaking padding oracle detection --- bbot/modules/lightfuzz_submodules/crypto.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index f93ddacc1..d855563f3 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -133,10 +133,11 @@ def possible_block_sizes(ciphertext_length): async def padding_oracle_execute(self, original_data, encoding, cookies, possible_first_byte=False): possible_block_sizes = self.possible_block_sizes(len(original_data)) - for block_size in possible_block_sizes: + for block_size in possible_block_sizes: ivblock = b"\x00" * block_size - datablock = b"\x00" * block_size + paddingblock = b"\x00" * block_size + datablock = original_data[-block_size:] if possible_first_byte: baseline_byte = b"\xFF" starting_pos = 0 @@ -145,16 +146,15 @@ async def padding_oracle_execute(self, original_data, encoding, cookies, possibl starting_pos = 1 baseline = self.compare_baseline( - self.event.data["type"], ivblock[:-1] + baseline_byte + datablock, cookies + self.event.data["type"], ivblock + paddingblock[:-1] + baseline_byte + datablock, cookies ) differ_count = 0 for i in range(starting_pos, starting_pos + 254): - byte = bytes([i]) oracle_probe = await self.compare_probe( baseline, self.event.data["type"], - self.format_agnostic_encode(ivblock[:-1] + byte + datablock, encoding), + self.format_agnostic_encode(ivblock + paddingblock[:-1] + byte + datablock, encoding), cookies, ) @@ -288,7 +288,9 @@ async def fuzz(self): {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text} ) - if confirmed_techniques: + if confirmed_techniques or ( + "padding" in truncate_probe[3].text.lower() or "padding" in mutate_probe[3].text.lower() + ): # Padding Oracle Test From 39ca3d5b3123c46d8937d575396652f4a9c38983 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 15:12:39 -0400 Subject: [PATCH 127/238] moderate refactor and test fix --- bbot/modules/lightfuzz_submodules/base.py | 17 +++++++++++++++++ bbot/modules/lightfuzz_submodules/cmdi.py | 9 ++------- bbot/modules/lightfuzz_submodules/crypto.py | 14 ++++++-------- bbot/modules/lightfuzz_submodules/path.py | 11 ++++------- bbot/modules/lightfuzz_submodules/serial.py | 14 +++++++++----- bbot/modules/lightfuzz_submodules/sqli.py | 7 +++++-- bbot/modules/lightfuzz_submodules/ssti.py | 2 +- .../module_tests/test_module_lightfuzz.py | 8 ++++---- 8 files changed, 48 insertions(+), 34 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 7ce5b0f34..5d15f1a19 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -156,3 +156,20 @@ async def standard_probe( retries=0, timeout=timeout, ) + + def metadata(self): + + metadata_string = f"Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]" + if self.event.data["original_value"] != "" and self.event.data["original_value"] != None: + metadata_string += ( + f" Original Value: [{self.lightfuzz.helpers.truncate_string(self.event.data['original_value'],200)}]" + ) + return metadata_string + + def probe_value(self, populate_empty=True): + + probe_value = str(self.event.data.get("original_value", "")) + if (probe_value is None or len(probe_value) == 0) and populate_empty == True: + probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + + return probe_value diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index 671574c95..92385c643 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -9,11 +9,7 @@ class CmdILightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - original_value = str(self.event.data.get("original_value", "")) - if original_value is not None and len(original_value) != 0: - probe_value = original_value - else: - probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + probe_value = self.probe_value() canary = self.lightfuzz.helpers.rand_string(8, numeric_only=True) http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) @@ -46,12 +42,11 @@ async def fuzz(self): except HttpCompareError as e: self.lightfuzz.debug(e) continue - if len(positive_detections) > 0: self.results.append( { "type": "FINDING", - "description": f"POSSIBLE OS Command Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [echo canary] CMD Probe Delimeters: [{' '.join(positive_detections)}]", + "description": f"POSSIBLE OS Command Injection. {self.metadata()} Detection Method: [echo canary] CMD Probe Delimeters: [{' '.join(positive_detections)}]", } ) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index d855563f3..7881620c9 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -188,7 +188,7 @@ async def padding_oracle(self, probe_value, cookies): { "type": "VULNERABILITY", "severity": "HIGH", - "description": f"Padding Oracle Vulnerability. Block size: [{str(block_size)}] Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]", + "description": f"Padding Oracle Vulnerability. Block size: [{str(block_size)}] {self.metadata()}", "context": context, } ) @@ -208,7 +208,7 @@ async def error_string_search(self, text_dict): self.results.append( { "type": "FINDING", - "description": f"Possible Cryptographic Error. Parameter: [{self.event.data['name']}] Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]", + "description": f"Possible Cryptographic Error. {self.metadata()} Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]", "context": context, } ) @@ -229,10 +229,8 @@ def identify_hash_function(hash_bytes): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - original_value = self.event.data.get("original_value", None) - if original_value is not None and original_value != "1": - probe_value = self.event.data["original_value"] - else: + probe_value = self.probe_value(populate_empty=False) + if not probe_value: self.lightfuzz.debug( f"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" ) @@ -278,7 +276,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"Probable Cryptographic Parameter [{self.event.data['name']}] Parameter Type [{self.event.data['type']}] Detection Technique(s): [{', '.join(confirmed_techniques)}]", + "description": f"Probable Cryptographic Parameter. {self.metadata()} Detection Technique(s): [{', '.join(confirmed_techniques)}]", "context": context, } ) @@ -323,7 +321,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"Possible {self.event.data['type']} parameter with {hash_instance.name.upper()} Hash as value. Parameter: [{self.event.data['name']}], linked to additional parameter [{additional_param_name}]", + "description": f"Possible {self.event.data['type']} parameter with {hash_instance.name.upper()} Hash as value. {self.metadata()}, linked to additional parameter [{additional_param_name}]", "context": context, } ) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index cae82b2da..1ac658deb 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -9,11 +9,8 @@ class PathTraversalLightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - - original_value = str(self.event.data.get("original_value", "")) - if original_value is not None and original_value != "1": - probe_value = urllib.parse.quote(original_value) - else: + probe_value = self.probe_value(populate_empty=False) + if not probe_value: self.lightfuzz.debug( f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" ) @@ -76,7 +73,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [{path_technique}]", + "description": f"POSSIBLE Path Traversal. {self.metadata()} Detection Method: [{path_technique}]", } ) # no need to report both techniques if they both work @@ -104,6 +101,6 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"POSSIBLE Path Traversal. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Absolute Path: {path}]", + "description": f"POSSIBLE Path Traversal. {self.metadata()} Detection Method: [Absolute Path: {path}]", } ) diff --git a/bbot/modules/lightfuzz_submodules/serial.py b/bbot/modules/lightfuzz_submodules/serial.py index 7ab6cfc5d..6081a550d 100644 --- a/bbot/modules/lightfuzz_submodules/serial.py +++ b/bbot/modules/lightfuzz_submodules/serial.py @@ -27,8 +27,8 @@ async def fuzz(self): "java.io.optionaldataexception", ] - original_value = self.event.data.get("original_value", None) - if not (original_value == None or original_value == ""): + probe_value = self.probe_value(populate_empty=False) + if probe_value: self.lightfuzz.debug( f"The Serialization Submodule only operates when there if no original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" ) @@ -39,11 +39,15 @@ async def fuzz(self): try: serialization_probe = await self.compare_probe(http_compare, self.event.data["type"], payload, cookies) if serialization_probe[0] == False and serialization_probe[1] != ["header"]: - if serialization_probe[3].status_code == 200 and "code" in serialization_probe[1]: + if ( + serialization_probe[3].status_code == 200 + and "code" in serialization_probe[1] + and "The requested URL was rejected" not in serialization_probe[3].text + ): self.results.append( { "type": "FINDING", - "description": f"POSSIBLE Unsafe Deserialization. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Technique: [Error Resolution] Serialization Payload: [{type}]", + "description": f"POSSIBLE Unsafe Deserialization. {self.metadata()} Technique: [Error Resolution] Serialization Payload: [{type}]", } ) elif serialization_probe[3].status_code == 500 or ( @@ -54,7 +58,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"POSSIBLE Unsafe Deserialization. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Technique: [Differential Error Analysis] Error-String: [{serialization_error}] Payload: [{type}]", + "description": f"POSSIBLE Unsafe Deserialization. {self.metadata()} Technique: [Differential Error Analysis] Error-String: [{serialization_error}] Payload: [{type}]", } ) break diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index c6a2f17ad..ec5848373 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -29,11 +29,14 @@ def evaluate_delay(self, mean_baseline, measured_delay): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) + + # custom probe_value generation if "original_value" in self.event.data and self.event.data["original_value"] is not None: probe_value = urllib.parse.quote(str(self.event.data["original_value"]), safe="") else: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + http_compare = self.compare_baseline( self.event.data["type"], probe_value, cookies, additional_params_populate_empty=True ) @@ -58,7 +61,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"Possible SQL Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Single Quote/Two Single Quote]", + "description": f"Possible SQL Injection. {self.metadata()} Detection Method: [Single Quote/Two Single Quote]", } ) except HttpCompareError as e: @@ -112,7 +115,7 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"Possible Blind SQL Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Delay Probe ({p})]", + "description": f"Possible Blind SQL Injection. {self.metadata()} Detection Method: [Delay Probe ({p})]", } ) diff --git a/bbot/modules/lightfuzz_submodules/ssti.py b/bbot/modules/lightfuzz_submodules/ssti.py index 9adc23adb..a4f8df08b 100644 --- a/bbot/modules/lightfuzz_submodules/ssti.py +++ b/bbot/modules/lightfuzz_submodules/ssti.py @@ -10,6 +10,6 @@ async def fuzz(self): self.results.append( { "type": "FINDING", - "description": f"POSSIBLE Server-side Template Injection. Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}] Detection Method: [Integer Multiplication]", + "description": f"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication]", } ) diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 1b42e2763..c1b10b8e6 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -59,7 +59,7 @@ def check(self, module_test, events): if e.type == "FINDING": if ( - "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [single-dot traversal tolerance" + "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Original Value: [default.jpg] Detection Method: [single-dot traversal tolerance (url-encoding)]" in e.data["description"] ): pathtraversal_finding_emitted = True @@ -110,7 +110,7 @@ def check(self, module_test, events): if e.type == "FINDING": if ( - "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Detection Method: [Absolute Path: /etc/passwd]" + "POSSIBLE Path Traversal. Parameter: [filename] Parameter Type: [GETPARAM] Original Value: [default.jpg] Detection Method: [Absolute Path: /etc/passwd]" in e.data["description"] ): pathtraversal_finding_emitted = True @@ -145,7 +145,7 @@ def request_handler(self, request): async def setup_after_prep(self, module_test): expect_args = {"method": "GET", "uri": "/"} - respond_args = {"response_data": "", "status": 302, "headers": {"Location": "/test?data=1"}} + respond_args = {"response_data": "", "status": 302, "headers": {"Location": "/test?data=9"}} module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) expect_args = re.compile("/test.*") @@ -162,7 +162,7 @@ def check(self, module_test, events): if e.type == "FINDING": if ( - "POSSIBLE Server-side Template Injection. Parameter: [data] Parameter Type: [GETPARAM] Detection Method: [Integer Multiplication]" + "POSSIBLE Server-side Template Injection. Parameter: [data] Parameter Type: [GETPARAM] Original Value: [9] Detection Method: [Integer Multiplication]" in e.data["description"] ): ssti_finding_emitted = True From 85c3c73014b6ecd3c3a33e1de58d7f9839169c26 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 15:29:49 -0400 Subject: [PATCH 128/238] small crypto bug fix --- bbot/modules/lightfuzz_submodules/crypto.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 7881620c9..3101becf4 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -306,7 +306,11 @@ async def fuzz(self): hash_instance = hash_function() # if there are any hash functions which match the length, we check the additional parameters to see if they cause identical changes # this would indicate they are being used to generate the hash - if hash_function and self.event.data["additional_params"]: + if ( + hash_function + and "additional_params" in self.event.data.keys() + and self.event.data["additional_params"] + ): for additional_param_name, additional_param_value in self.event.data["additional_params"].items(): additional_param_probe = await self.compare_probe( http_compare, From 6875f5622eaef06620fb0e222d458c79a1603682 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 15:39:15 -0400 Subject: [PATCH 129/238] better error handling --- bbot/modules/lightfuzz_submodules/crypto.py | 30 ++++++++++++++------- bbot/modules/lightfuzz_submodules/sqli.py | 2 +- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 3101becf4..a644f87d9 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -254,9 +254,15 @@ async def fuzz(self): http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) # Cryptographic Response Divergence Test - arbitrary_probe = await self.compare_probe(http_compare, self.event.data["type"], "AAAAAAA", cookies) - truncate_probe = await self.compare_probe(http_compare, self.event.data["type"], truncate_probe_value, cookies) - mutate_probe = await self.compare_probe(http_compare, self.event.data["type"], mutate_probe_value, cookies) + try: + arbitrary_probe = await self.compare_probe(http_compare, self.event.data["type"], "AAAAAAA", cookies) + truncate_probe = await self.compare_probe( + http_compare, self.event.data["type"], truncate_probe_value, cookies + ) + mutate_probe = await self.compare_probe(http_compare, self.event.data["type"], mutate_probe_value, cookies) + except HttpCompareError as e: + self.lightfuzz.warning(f"Encountered HttpCompareError Sending Compare Probe: {e}") + return confirmed_techniques = [] if mutate_probe[0] == False and "body" in mutate_probe[1]: @@ -312,13 +318,17 @@ async def fuzz(self): and self.event.data["additional_params"] ): for additional_param_name, additional_param_value in self.event.data["additional_params"].items(): - additional_param_probe = await self.compare_probe( - http_compare, - self.event.data["type"], - probe_value, - cookies, - additional_params_override={additional_param_name: additional_param_value + "A"}, - ) + try: + additional_param_probe = await self.compare_probe( + http_compare, + self.event.data["type"], + probe_value, + cookies, + additional_params_override={additional_param_name: additional_param_value + "A"}, + ) + except HttpCompareError as e: + self.lightfuzz.warning(f"Encountered HttpCompareError Sending Compare Probe: {e}") + continue # the additional parameter affects the potential hash parameter (suggesting its calculated in the hash) if additional_param_probe[0] == False and (additional_param_probe[1] == mutate_probe[1]): context = f"Lightfuzz Cryptographic Probe Submodule detected a parameter ({self.event.data['name']}) that is a likely a hash, which is connected to another parameter {additional_param_name})" diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index ec5848373..b89789d01 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -65,7 +65,7 @@ async def fuzz(self): } ) except HttpCompareError as e: - self.lightfuzz.critical(e) + self.lightfuzz.warning(f"Encountered HttpCompareError Sending Compare Probe: {e}") standard_probe_strings = [ f"'||pg_sleep({str(self.expected_delay)})--", # postgres From 10c11b35f25606ab8a66594bebfc93957762967e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 15:59:40 -0400 Subject: [PATCH 130/238] updating hunt to show more detail in finding --- bbot/modules/hunt.py | 8 +++++++- bbot/test/test_step_2/module_tests/test_module_hunt.py | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index 649eaaa19..baa2bd905 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -286,7 +286,13 @@ async def handle_event(self, event): p = event.data["name"] for k in hunt_param_dict.keys(): if p.lower() in hunt_param_dict[k]: - description = f"Found potential {k.upper()} parameter [{p}]" + + description = f"Found potential {k.upper()} parameter: [{p}] Parameter Type: [{event.data['type']}]" + if event.data["original_value"] != "" and event.data["original_value"] != None: + description += ( + f" Original Value: [{self.helpers.truncate_string(event.data['original_value'],200)}]" + ) + data = {"host": str(event.host), "description": description} url = event.data.get("url", "") if url: diff --git a/bbot/test/test_step_2/module_tests/test_module_hunt.py b/bbot/test/test_step_2/module_tests/test_module_hunt.py index ff5eed716..7914eb26b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_hunt.py +++ b/bbot/test/test_step_2/module_tests/test_module_hunt.py @@ -14,7 +14,10 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): + assert any( - e.type == "FINDING" and e.data["description"] == "Found potential INSECURE CRYPTOGRAPHY parameter [cipher]" + e.type == "FINDING" + and e.data["description"] + == "Found potential INSECURE CRYPTOGRAPHY parameter: [cipher] Parameter Type: [GETPARAM] Original Value: [xor]" for e in events ) From 09267731f843eaac426329feff99be6b24d4e101 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 16:02:27 -0400 Subject: [PATCH 131/238] little hunt bug fix --- bbot/modules/hunt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index baa2bd905..5de1bdfff 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -288,7 +288,11 @@ async def handle_event(self, event): if p.lower() in hunt_param_dict[k]: description = f"Found potential {k.upper()} parameter: [{p}] Parameter Type: [{event.data['type']}]" - if event.data["original_value"] != "" and event.data["original_value"] != None: + if ( + "original_value" in event.data.keys() + and event.data["original_value"] != "" + and event.data["original_value"] != None + ): description += ( f" Original Value: [{self.helpers.truncate_string(event.data['original_value'],200)}]" ) From f1d0096a054432f30366056376baad7f02b7662a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 16:07:42 -0400 Subject: [PATCH 132/238] missing import --- bbot/modules/lightfuzz_submodules/crypto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index a644f87d9..79d50fe17 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -1,6 +1,7 @@ import base64 import hashlib from .base import BaseLightfuzz +from bbot.errors import HttpCompareError from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote, quote From 0ae5ad8085acda787c90207d3b52c64f8cf552c2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 15 Aug 2024 23:33:28 -0400 Subject: [PATCH 133/238] adding no-post safety option --- bbot/modules/lightfuzz.py | 9 +++++- .../module_tests/test_module_lightfuzz.py | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 1d0737218..f6cd45615 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -32,10 +32,11 @@ class lightfuzz(BaseModule): "serial": {"description": "Unsafe Deserialization Probe", "module": SerialLightfuzz}, } - options = {"force_common_headers": False, "enabled_submodules": []} + options = {"force_common_headers": False, "enabled_submodules": [], "disable_post": False} options_desc = { "force_common_headers": "Force emit commonly exploitable parameters that may be difficult to detect", "enabled_submodules": "A list of submodules to enable. Empty list enabled all modules.", + "disable_post": "Disable processing of POST parameters, avoiding form submissions.", } meta = {"description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner"} @@ -62,6 +63,7 @@ async def setup(self): self.event_dict = {} self.interactsh_subdomain_tags = {} self.interactsh_instance = None + self.disable_post = self.config.get("disable_post", False) self.enabled_submodules = self.config.get("enabled_submodules") for m in self.enabled_submodules: @@ -206,3 +208,8 @@ async def finish(self): await self.interactsh_callback(r) except InteractshError as e: self.debug(f"Error in interact.sh: {e}") + + async def filter_event(self, event): + if event.type == "WEB_PARAMETER" and self.disable_post and event.data["type"] == "POSTPARAM": + return False, "POST parameter disabled in lilghtfuzz module" + return True diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index c1b10b8e6..36ac57f29 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -489,6 +489,38 @@ def check(self, module_test, events): assert sqli_finding_emitted, "SQLi Single/Double Quote postparam FINDING not emitted" +# disable_post test +class Test_Lightfuzz_disable_post(Test_Lightfuzz_sqli_post): + + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "enabled_submodules": ["sqli"], + "disable_post": True, + } + }, + } + + def check(self, module_test, events): + web_parameter_emitted = False + sqli_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "Possible SQL Injection. Parameter: [search] Parameter Type: [POSTPARAM] Detection Method: [Single Quote/Two Single Quote]" + in e.data["description"] + ): + sqli_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert not sqli_finding_emitted, "post-based SQLI emitted despite post-parameters being disabled" + + # SQLI Single Quote/Two Single Quote (headers) class Test_Lightfuzz_sqli_headers(Test_Lightfuzz_sqli): From 89b02125b4d2e190dcfefb69fbf077f15d86c040 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 18 Aug 2024 12:38:44 -0400 Subject: [PATCH 134/238] false positive reduction for path module --- bbot/modules/lightfuzz_submodules/path.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index 1ac658deb..d4834221d 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -64,6 +64,7 @@ async def fuzz(self): and doubledot_probe[0] == False and doubledot_probe[3] != None and doubledot_probe[1] != ["header"] + and "The requested URL was rejected" not in doubledot_probe[3].text ): confirmations += 1 self.lightfuzz.verbose( From e819c5d9d49995575d07c1cbaa35e102e4d48053 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 18 Aug 2024 12:42:27 -0400 Subject: [PATCH 135/238] enforcing prefix blacklist --- bbot/modules/internal/excavate.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 6380a2a87..c0cff7e16 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -328,6 +328,11 @@ class excavateTestRule(ExcavateRule): "PHPSESSID", ] + parameter_blacklist_prefix = [ + "TS01" # Big-IP F5 Persistence Cookies + "BIGipServerpool_" + ] + yara_rule_name_regex = re.compile(r"rule\s(\w+)\s{") yara_rule_regex = re.compile(r"(?s)((?:rule\s+\w+\s*{[^{}]*(?:{[^{}]*}[^{}]*)*[^{}]*(?:/\S*?}[^/]*?/)*)*})") @@ -336,6 +341,10 @@ def in_bl(self, value): for bl_param in self.parameter_blacklist: if bl_param.lower() == value.lower(): in_bl = True + + for bl_param_prefix in self.parameter_blacklist_prefix: + if value.lower().startswith(bl_param_prefix.lower()): + in_bl = True return in_bl def url_unparse(self, param_type, parsed_url): From 4e5789e01743ca23681a54142ccf51129bce0861 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 18 Aug 2024 12:49:46 -0400 Subject: [PATCH 136/238] lightfuzz presets rework --- bbot/presets/web/lightfuzz-intense.yml | 6 ++++-- bbot/presets/web/lightfuzz-max.yml | 23 +++++++++++++++++++++++ bbot/presets/web/lightfuzz.yml | 8 +++++--- 3 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 bbot/presets/web/lightfuzz-max.yml diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml index 23b5a27f1..22fa02fc8 100644 --- a/bbot/presets/web/lightfuzz-intense.yml +++ b/bbot/presets/web/lightfuzz-intense.yml @@ -8,14 +8,16 @@ modules: - lightfuzz - robots - badsecrets + - hunt config: url_querystring_remove: False - url_querystring_collapse: False + url_querystring_collapse: True web_spider_distance: 4 web_spider_depth: 5 modules: lightfuzz: - force_common_headers: True + force_common_headers: False + submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] excavate: retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-max.yml b/bbot/presets/web/lightfuzz-max.yml new file mode 100644 index 000000000..c0ede2fbb --- /dev/null +++ b/bbot/presets/web/lightfuzz-max.yml @@ -0,0 +1,23 @@ +description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques + +flags: + - web-paramminer + +modules: + - httpx + - lightfuzz + - robots + - badsecrets + - hunt + +config: + url_querystring_remove: False + url_querystring_collapse: False + web_spider_distance: 4 + web_spider_depth: 5 + modules: + lightfuzz: + force_common_headers: True + submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] + excavate: + retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml index f8deb23c5..277571eb5 100644 --- a/bbot/presets/web/lightfuzz.yml +++ b/bbot/presets/web/lightfuzz.yml @@ -4,13 +4,15 @@ modules: - httpx - lightfuzz - badsecrets + - hunt config: url_querystring_remove: False - web_spider_distance: 4 - web_spider_depth: 5 + web_spider_distance: 3 + web_spider_depth: 4 modules: lightfuzz: - submodules_enabled: [cmdi,crypto,sqli,ssti,xss] + submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] + disable_post: True excavate: retain_querystring: True \ No newline at end of file From 95323a0d1e9ce4d7fff113b75f83c5ea8b30fa8c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 18 Aug 2024 13:27:25 -0400 Subject: [PATCH 137/238] ugh --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index c0cff7e16..a21576734 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -1012,7 +1012,7 @@ async def handle_event(self, event): cookie_name = header_value.split("=")[0] cookie_value = header_value.split("=")[1].split(";")[0] - if self.in_bl(cookie_value) == False: + if self.in_bl(cookie_name) == False: self.assigned_cookies[cookie_name] = cookie_value description = f"Set-Cookie Assigned Cookie [{cookie_name}]" data = { From 1128ab7aed075ba5ec0e824aca5a1f33212aba6a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sun, 18 Aug 2024 14:03:18 -0400 Subject: [PATCH 138/238] bug fix/test --- bbot/modules/internal/excavate.py | 2 +- .../module_tests/test_module_excavate.py | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index a21576734..26d7f3e23 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -329,7 +329,7 @@ class excavateTestRule(ExcavateRule): ] parameter_blacklist_prefix = [ - "TS01" # Big-IP F5 Persistence Cookies + "TS01", # Big-IP F5 Persistence Cookies "BIGipServerpool_" ] diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 0065568b9..84aba05e1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -902,3 +902,43 @@ def check(self, module_test, events): assert found_first_cookie == True assert found_second_cookie == True + + +class TestExcavateHeaders_blacklist(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["excavate", "httpx"] + config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} + + async def setup_before_prep(self, module_test): + + module_test.httpserver.expect_request("/").respond_with_data( + "

test

", + status=200, + headers={ + "Set-Cookie": [ + "COOKIE1=aaaa; Secure; HttpOnly", + "TS0113CC91=bbbb; Secure; HttpOnly; SameSite=None", + "PHPSESSID=cccc; Secure; HttpOnly; SameSite=None", + ] + }, + ) + + def check(self, module_test, events): + + found_first_cookie = False + found_second_cookie = False + found_third_cookie = False + + for e in events: + if e.type == "WEB_PARAMETER": + if e.data["name"] == "COOKIE1": + found_first_cookie = True + if e.data["name"] == "PHPSESSID": + found_second_cookie = True + if e.data["name"] == "TS0113CC91": + found_third_cookie = True + + assert found_first_cookie == True + assert found_second_cookie == False + assert found_third_cookie == False From 16cdea35fddbe8ec72a67e27ddc0e56093131707 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 22 Aug 2024 23:52:36 -0400 Subject: [PATCH 139/238] fixing issues with serializtion regexes --- bbot/modules/internal/excavate.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 26d7f3e23..37a4180ed 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -619,13 +619,13 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte class SerializationExtractor(ExcavateRule): regexes = { - "Java": re.compile(r"[^a-zA-Z0-9\/+]rO0[a-zA-Z0-9+\/]+={0,2}"), - "Ruby": re.compile(r"[^a-zA-Z0-9\/+]BAh[a-zA-Z0-9+\/]+={0,2}"), - "DOTNET": re.compile(r"[^a-zA-Z0-9\/+]AAEAAAD\/\/[a-zA-Z0-9\/+]+={0,2}"), - "PHP_Array": re.compile(r"[^a-zA-Z0-9\/+]YTo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), - "PHP_String": re.compile(r"[^a-zA-Z0-9\/+]czo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), - "PHP_Object": re.compile(r"[^a-zA-Z0-9\/+]Tzo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), - "Possible_Compressed": re.compile(r"[^a-zA-Z0-9\/+]H4sIAAAAAAAA[a-zA-Z0-9+\/]+={0,2}"), + "Java": re.compile(r"[^a-zA-Z0-9\/+][\"']?rO0[a-zA-Z0-9+\/]+={0,2}"), + "Ruby": re.compile(r"[^[^a-zA-Z0-9\/+][\"']?BAh[a-zA-Z0-9+\/]+={0,2}"), + "DOTNET": re.compile(r"[^a-zA-Z0-9\/+][\"']?AAEAAAD\/\/[a-zA-Z0-9\/+]+={0,2}"), + "PHP_Array": re.compile(r"[^a-zA-Z0-9\/+][\"']?YTo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), + "PHP_String": re.compile(r"[^a-zA-Z0-9\/+][\"']?czo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), + "PHP_Object": re.compile(r"[^a-zA-Z0-9\/+][\"']?Tzo[xyz0123456][a-zA-Z0-9+\/]+={0,2}"), + "Possible_Compressed": re.compile(r"[^a-zA-Z0-9\/+][\"']?H4sIAAAA[a-zA-Z0-9+\/]+={0,2}"), } yara_rules = {} From bfa22d53f915322bbe448e2b217e5dc468ffa906 Mon Sep 17 00:00:00 2001 From: Paul Mueller Date: Fri, 23 Aug 2024 10:17:54 -0400 Subject: [PATCH 140/238] Update lightfuzz.yml --- bbot/presets/web/lightfuzz.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml index 277571eb5..8466d4a8e 100644 --- a/bbot/presets/web/lightfuzz.yml +++ b/bbot/presets/web/lightfuzz.yml @@ -1,4 +1,4 @@ -description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques +description: Discovery web parameters and lightly fuzz them for vulnerabilities, without some of the more intense discovery techniques modules: - httpx @@ -15,4 +15,4 @@ config: submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] disable_post: True excavate: - retain_querystring: True \ No newline at end of file + retain_querystring: True From 6df6163d0cea3783d2401241df66de7b6ee2d6c0 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 5 Sep 2024 16:42:56 -0400 Subject: [PATCH 141/238] adding cookie test --- bbot/test/test_step_1/test_web.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index a181306a1..10ea938f6 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -442,3 +442,24 @@ async def test_web_cookies(bbot_scanner, httpx_mock): assert not client2.cookies await scan._cleanup() + + +@pytest.mark.asyncio +async def test_http_sendcookies(bbot_scanner, bbot_httpserver): + endpoint = "/" + url = bbot_httpserver.url_for(endpoint) + from werkzeug.wrappers import Response + + def echo_cookies_handler(request): + cookies = request.cookies + cookie_str = "; ".join([f"{key}={value}" for key, value in cookies.items()]) + return Response(f"Echoed Cookies: {cookie_str}\nEchoed Headers: {request.headers}") + + bbot_httpserver.expect_request(uri=endpoint).respond_with_handler(echo_cookies_handler) + scan1 = bbot_scanner("127.0.0.1", config={"web": {"debug": True}}) + r1 = await scan1.helpers.request(url, cookies={"foo": "bar"}) + print(r1.text) + + assert r1 is not None, "Request to self-signed SSL server went through even with ssl_verify=True" + assert "bar" in r1.text + await scan1._cleanup() From bcbfe238b7fe5899711eae8b8bb92d0db6511aa8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 5 Sep 2024 17:34:30 -0400 Subject: [PATCH 142/238] removing extranious blacklist --- bbot/modules/internal/excavate.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index c25702f73..5e16507f2 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -313,21 +313,8 @@ class excavateTestRule(ExcavateRule): _module_threads = 8 - parameter_blacklist = [ - "__VIEWSTATE", - "__EVENTARGUMENT", - "__EVENTVALIDATION", - "__EVENTTARGET", - "__EVENTARGUMENT", - "__VIEWSTATEGENERATOR", - "__SCROLLPOSITIONY", - "__SCROLLPOSITIONX", - "ASP.NET_SessionId", - "JSESSIONID", - "PHPSESSID", - ] - parameter_blacklist_prefix = ["TS01", "BIGipServerpool_"] # Big-IP F5 Persistence Cookies + parameter_blacklist = set( p.lower() for p in [ From 0473a5174ae923badba3ad78239e84d189fd56d9 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 5 Sep 2024 18:02:52 -0400 Subject: [PATCH 143/238] removing unnecessary blacklist, fixing tests --- bbot/modules/lightfuzz.py | 21 ------------------- .../module_tests/test_module_excavate.py | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index f6cd45615..14a319841 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -41,20 +41,6 @@ class lightfuzz(BaseModule): meta = {"description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner"} common_headers = ["x-forwarded-for", "user-agent"] - parameter_blacklist = [ - "__VIEWSTATE", - "__EVENTARGUMENT", - "__EVENTVALIDATION", - "__EVENTTARGET", - "__EVENTARGUMENT", - "__VIEWSTATEGENERATOR", - "__SCROLLPOSITIONY", - "__SCROLLPOSITIONX", - "ASP.NET_SessionId", - "JSESSIONID", - "PHPSESSID", - "__cf_bm", - ] in_scope_only = True _module_threads = 4 @@ -121,13 +107,6 @@ def _outgoing_dedup_hash(self, event): ) ) - def in_bl(self, value): - in_bl = False - for bl_param in self.parameter_blacklist: - if bl_param.lower() == value.lower(): - in_bl = True - return in_bl - def url_unparse(self, param_type, parsed_url): if param_type == "GETPARAM": querystring = "" diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index fb5aeee0e..1f0c5c516 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -1906,7 +1906,7 @@ def check(self, module_test, events): class TestExcavateHeaders(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] - modules_overrides = ["excavate", "httpx"] + modules_overrides = ["excavate", "httpx", "hunt"] config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} async def setup_before_prep(self, module_test): From 4836f1c8a5103816f27808d54cbaab2131d5ac30 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 10:29:30 -0400 Subject: [PATCH 144/238] original value display bug fix --- bbot/modules/hunt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index 5de1bdfff..6de99a24f 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -294,7 +294,7 @@ async def handle_event(self, event): and event.data["original_value"] != None ): description += ( - f" Original Value: [{self.helpers.truncate_string(event.data['original_value'],200)}]" + f" Original Value: [{self.helpers.truncate_string(str(event.data['original_value']),200)}]" ) data = {"host": str(event.host), "description": description} From aedb391bb1a012578af960c8767dd2051efa4ead Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 10:35:32 -0400 Subject: [PATCH 145/238] black --- bbot/modules/internal/excavate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index bf34ecb88..04c7c731a 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -314,7 +314,7 @@ class excavateTestRule(ExcavateRule): _module_threads = 8 parameter_blacklist_prefix = ["TS01", "BIGipServerpool_"] # Big-IP F5 Persistence Cookies - + parameter_blacklist = set( p.lower() for p in [ From a3cf59250953aa2827f50df385bd496d5e6514c1 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 11:03:56 -0400 Subject: [PATCH 146/238] flake8 nightmare --- bbot/modules/internal/excavate.py | 2 +- bbot/modules/lightfuzz.py | 11 +- bbot/modules/lightfuzz_submodules/crypto.py | 2 +- bbot/modules/lightfuzz_submodules/sqli.py | 1 - .../module_tests/test_module_excavate.py | 836 +----------------- .../module_tests/test_module_lightfuzz.py | 8 - 6 files changed, 28 insertions(+), 832 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 04c7c731a..481b63f58 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -7,7 +7,7 @@ from bbot.errors import ExcavateError import bbot.core.helpers.regexes as bbot_regexes from bbot.modules.internal.base import BaseInternalModule -from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, quote +from urllib.parse import urlparse, urljoin, parse_qs, urlunparse def find_subclasses(obj, base_class): diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 14a319841..8136ee2d9 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -1,12 +1,7 @@ from bbot.modules.base import BaseModule -import statistics -import re -import os -import base64 -import urllib.parse - -from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote -from bbot.errors import InteractshError, HttpCompareError + +from urllib.parse import urlunparse +from bbot.errors import InteractshError from .lightfuzz_submodules.cmdi import CmdILightfuzz from .lightfuzz_submodules.crypto import CryptoLightfuzz diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 79d50fe17..9141c240d 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -2,7 +2,7 @@ import hashlib from .base import BaseLightfuzz from bbot.errors import HttpCompareError -from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, unquote, quote +from urllib.parse import unquote, quote class CryptoLightfuzz(BaseLightfuzz): diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index b89789d01..664fca92b 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -73,7 +73,6 @@ async def fuzz(self): f"' AND (SELECT FROM DBMS_LOCK.SLEEP({str(self.expected_delay)})) AND '1'='1" # oracle (not tested) f"; WAITFOR DELAY '00:00:{str(self.expected_delay)}'--", # mssql (not tested) ] - method = "GET" baseline_1 = await self.standard_probe( self.event.data["type"], cookies, probe_value, additional_params_populate_empty=True diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index a6708255c..71ab9fd64 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -133,7 +133,7 @@ async def setup_before_prep(self, module_test): def check(self, module_test, events): root_relative_detection = False page_relative_detection_1 = False - page_relative_detection_1 = False + page_relative_detection_2 = False root_page_confusion_1 = False root_page_confusion_2 = False @@ -462,14 +462,21 @@ class TestExcavateParameterExtraction(TestExcavate):

Use the form below to submit a GET request:

-

+

Simple POST Form

Use the form below to submit a POST request:

-

+

+ +
+

Simple Generic Form

+

Use the form below to submit a request:

+
+ +

Links

@@ -488,15 +495,19 @@ def check(self, module_test, events): found_jquery_post = False found_form_get = False found_form_post = False + found_form_generic = False found_jquery_get_original_value = False found_jquery_post_original_value = False found_form_get_original_value = False found_form_post_original_value = False + found_form_generic_original_value = False found_htmltags_a = False found_htmltags_img = False for e in events: + if e.type == "WEB_PARAMETER": + if e.data["description"] == "HTTP Extracted Parameter [jqueryget] (GET jquery Submodule)": found_jquery_get = True if e.data["original_value"] == "value1": @@ -507,16 +518,21 @@ def check(self, module_test, events): if e.data["original_value"] == "value2": found_jquery_post_original_value = True - if e.data["description"] == "HTTP Extracted Parameter [q] (GET Form Submodule)": + if e.data["description"] == "HTTP Extracted Parameter [q1] (GET Form Submodule)": found_form_get = True if e.data["original_value"] == "flowers": found_form_get_original_value = True - if e.data["description"] == "HTTP Extracted Parameter [q] (POST Form Submodule)": + if e.data["description"] == "HTTP Extracted Parameter [q2] (POST Form Submodule)": found_form_post = True if e.data["original_value"] == "boats": found_form_post_original_value = True + if e.data["description"] == "HTTP Extracted Parameter [q3] (Generic Form Submodule)": + found_form_generic = True + if e.data["original_value"] == "candles": + found_form_generic_original_value = True + if e.data["description"] == "HTTP Extracted Parameter [age] (HTML Tags Submodule)": if e.data["original_value"] == "456": if "id" in e.data["additional_params"].keys(): @@ -531,10 +547,12 @@ def check(self, module_test, events): assert found_jquery_post, "Did not extract Jquery POST parameters" assert found_form_get, "Did not extract Form GET parameters" assert found_form_post, "Did not extract Form POST parameters" + assert found_form_generic, "Did not extract Form (Generic) parameters" assert found_jquery_get_original_value, "Did not extract Jquery GET parameter original_value" assert found_jquery_post_original_value, "Did not extract Jquery POST parameter original_value" assert found_form_get_original_value, "Did not extract Form GET parameter original_value" assert found_form_post_original_value, "Did not extract Form POST parameter original_value" + assert found_form_generic_original_value, "Did not extract Form (Generic) parameter original_value" assert found_htmltags_a, "Did not extract parameter(s) from a-tag" assert found_htmltags_img, "Did not extract parameter(s) from img-tag" @@ -1036,8 +1054,6 @@ def check(self, module_test, events): from .base import ModuleTestBase, tempwordlist from bbot.modules.internal.excavate import ExcavateRule -import yara - class TestExcavate(ModuleTestBase): targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/links.html"] @@ -1132,812 +1148,6 @@ def check(self, module_test, events): ) -class TestExcavate2(TestExcavate): - targets = ["http://127.0.0.1:8888/", "test.notreal", "http://127.0.0.1:8888/subdir/"] - - async def setup_before_prep(self, module_test): - # root relative - expect_args = {"method": "GET", "uri": "/rootrelative.html"} - respond_args = {"response_data": "alive"} - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - # page relative - expect_args = {"method": "GET", "uri": "/subdir/pagerelative.html"} - respond_args = {"response_data": "alive"} - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - expect_args = {"method": "GET", "uri": "/subdir/"} - respond_args = { - "response_data": """ - root relative - page relative 1 - page relative 2 - """ - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - module_test.httpserver.no_handler_status_code = 404 - - def check(self, module_test, events): - root_relative_detection = False - page_relative_detection_1 = False - page_relative_detection_1 = False - root_page_confusion_1 = False - root_page_confusion_2 = False - - for e in events: - if e.type == "URL_UNVERIFIED": - # these cases represent the desired behavior for parsing relative links - if e.data == "http://127.0.0.1:8888/rootrelative.html": - root_relative_detection = True - if e.data == "http://127.0.0.1:8888/subdir/pagerelative1.html": - page_relative_detection_1 = True - if e.data == "http://127.0.0.1:8888/subdir/pagerelative2.html": - page_relative_detection_2 = True - - # these cases indicates that excavate parsed the relative links incorrectly - if e.data == "http://127.0.0.1:8888/pagerelative.html": - root_page_confusion_1 = True - if e.data == "http://127.0.0.1:8888/subdir/rootrelative.html": - root_page_confusion_2 = True - - assert root_relative_detection, "Failed to properly excavate root-relative URL" - assert page_relative_detection_1, "Failed to properly excavate page-relative URL" - assert page_relative_detection_2, "Failed to properly excavate page-relative URL" - assert not root_page_confusion_1, "Incorrectly detected page-relative URL" - assert not root_page_confusion_2, "Incorrectly detected root-relative URL" - - -class TestExcavateRedirect(TestExcavate): - targets = ["http://127.0.0.1:8888/", "http://127.0.0.1:8888/relative/", "http://127.0.0.1:8888/nonhttpredirect/"] - config_overrides = {"scope": {"report_distance": 1}} - - async def setup_before_prep(self, module_test): - # absolute redirect - module_test.httpserver.expect_request("/").respond_with_data( - "", status=302, headers={"Location": "https://www.test.notreal/yep"} - ) - module_test.httpserver.expect_request("/relative/").respond_with_data( - "", status=302, headers={"Location": "./owa/"} - ) - module_test.httpserver.expect_request("/relative/owa/").respond_with_data( - "ftp://127.0.0.1:2121\nsmb://127.0.0.1\nssh://127.0.0.2" - ) - module_test.httpserver.expect_request("/nonhttpredirect/").respond_with_data( - "", status=302, headers={"Location": "awb://127.0.0.1:7777"} - ) - module_test.httpserver.no_handler_status_code = 404 - - def check(self, module_test, events): - - assert 1 == len( - [ - e - for e in events - if e.type == "URL_UNVERIFIED" and e.data == "https://www.test.notreal/yep" and e.scope_distance == 1 - ] - ) - assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/relative/owa/"]) - assert 1 == len( - [ - e - for e in events - if e.type == "FINDING" and e.data["description"] == "Non-HTTP URI: awb://127.0.0.1:7777" - ] - ) - assert 1 == len( - [ - e - for e in events - if e.type == "PROTOCOL" and e.data["protocol"] == "AWB" and e.data.get("port", 0) == 7777 - ] - ) - assert 1 == len( - [ - e - for e in events - if e.type == "FINDING" and e.data["description"] == "Non-HTTP URI: ftp://127.0.0.1:2121" - ] - ) - assert 1 == len( - [ - e - for e in events - if e.type == "PROTOCOL" and e.data["protocol"] == "FTP" and e.data.get("port", 0) == 2121 - ] - ) - assert 1 == len( - [e for e in events if e.type == "FINDING" and e.data["description"] == "Non-HTTP URI: smb://127.0.0.1"] - ) - assert 1 == len( - [e for e in events if e.type == "PROTOCOL" and e.data["protocol"] == "SMB" and not "port" in e.data] - ) - assert 0 == len([e for e in events if e.type == "FINDING" and "ssh://127.0.0.1" in e.data["description"]]) - assert 0 == len([e for e in events if e.type == "PROTOCOL" and e.data["protocol"] == "SSH"]) - - -class TestExcavateQuerystringRemoveTrue(TestExcavate): - targets = ["http://127.0.0.1:8888/"] - config_overrides = {"url_querystring_remove": True, "url_querystring_collapse": True} - lots_of_params = """ - - - - - - - - - - - """ - - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data(self.lots_of_params) - - def check(self, module_test, events): - assert len([e for e in events if e.type == "URL_UNVERIFIED"]) == 2 - assert ( - len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.1:8888/endpoint"]) == 1 - ) - - -class TestExcavateQuerystringRemoveFalse(TestExcavateQuerystringRemoveTrue): - config_overrides = {"url_querystring_remove": False, "url_querystring_collapse": True} - - def check(self, module_test, events): - assert ( - len( - [ - e - for e in events - if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?") - ] - ) - == 1 - ) - - -class TestExcavateQuerystringCollapseFalse(TestExcavateQuerystringRemoveTrue): - config_overrides = {"url_querystring_remove": False, "url_querystring_collapse": False} - - def check(self, module_test, events): - assert ( - len( - [ - e - for e in events - if e.type == "URL_UNVERIFIED" and e.data.startswith("http://127.0.0.1:8888/endpoint?") - ] - ) - == 10 - ) - - -class TestExcavateMaxLinksPerPage(TestExcavate): - targets = ["http://127.0.0.1:8888/"] - config_overrides = {"web": {"spider_links_per_page": 10, "spider_distance": 1}} - - lots_of_links = """ - - - - - - - - - - - - - - - - - - - - - - - - - - """ - - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data(self.lots_of_links) - - def check(self, module_test, events): - url_unverified_events = [e for e in events if e.type == "URL_UNVERIFIED"] - # base URL + 25 links = 26 - assert len(url_unverified_events) == 26 - url_data = [e.data for e in url_unverified_events if "spider-max" not in e.tags and "spider-danger" in e.tags] - assert len(url_data) >= 10 and len(url_data) <= 12 - url_events = [e for e in events if e.type == "URL"] - assert len(url_events) == 11 - - -class TestExcavateCSP(TestExcavate): - - csp_test_header = "default-src 'self'; script-src asdf.test.notreal; object-src 'none';" - - async def setup_before_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/"} - respond_args = {"headers": {"Content-Security-Policy": self.csp_test_header}} - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check(self, module_test, events): - assert any(e.data == "asdf.test.notreal" for e in events) - - -class TestExcavateURL(TestExcavate): - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data( - "SomeSMooshedDATAhttps://asdffoo.test.notreal/some/path" - ) - - def check(self, module_test, events): - assert any(e.data == "asdffoo.test.notreal" for e in events) - assert any(e.data == "https://asdffoo.test.notreal/some/path" for e in events) - - -class TestExcavateURL_IP(TestExcavate): - - targets = ["http://127.0.0.1:8888/", "127.0.0.2"] - - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data("SomeSMooshedDATAhttps://127.0.0.2/some/path") - - def check(self, module_test, events): - assert any(e.data == "127.0.0.2" for e in events) - assert any(e.data == "https://127.0.0.2/some/path" for e in events) - - -class TestExcavateSerializationNegative(TestExcavate): - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data( - "

llsdtVVFlJxhcGGYTo2PMGTRNFVKZxeKTVbhyosM3Sm/5apoY1/yUmN6HVcn+Xt798SPzgXQlZMttsqp1U1iJFmFO2aCGL/v3tmm/fs7itYsoNnJCelWvm9P4ic1nlKTBOpMjT5B5NmriZwTAzZ5ASjCKcmN8Vh=

" - ) - - def check(self, module_test, events): - assert not any(e.type == "FINDING" for e in events), "Found Results without word boundary" - - -class TestExcavateSerializationPositive(TestExcavate): - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data( - """ -

.NET

-

AAEAAAD/////AQAAAAAAAAAMAgAAAFJTeXN0ZW0uQ29sbGVjdGlvbnMuR2VuZXJpYy5MaXN0YDFbW1N5c3RlbS5TdHJpbmddXSwgU3lzdGVtLCBWZXJzaW9uPTQuMC4wLjAsIEN1bHR1cmU9bmV1dHJhbCwgUHVibGljS2V5VG9rZW49YjAzZjVmN2YxMWQ1MGFlMwEAAAAIQ29tcGFyZXIQSXRlbUNvdW50AQMAAAAJAwAAAAlTeXN0ZW0uU3RyaW5nW10FAAAACQIAAAAJBAAAAAkFAAAACRcAAAAJCgAAAAkLAAAACQwAAAAJDQAAAAkOAAAACQ8AAAAJEAAAAAkRAAAACRIAAAAJEwAAAA==

-

Java

-

rO0ABXQADUhlbGxvLCB3b3JsZCE=

-

PHP (string)

-

czoyNDoiSGVsbG8sIHdvcmxkISBNb3JlIHRleHQuIjs=

-

PHP (array)

-

YTo0OntpOjA7aToxO2k6MTtzOjE0OiJzZWNvbmQgZWxlbWVudCI7aToyO2k6MztpOjM7czoxODoiTW9yZSB0ZXh0IGluIGFycmF5Ijt9

-

PHP (object)

-

TzoxMjoiU2FtcGxlT2JqZWN0IjoyOntzOjg6InByb3BlcnR5IjtzOjEzOiJJbml0aWFsIHZhbHVlIjtzOjE2OiJhZGRpdGlvbmFsU3RyaW5nIjtzOjIxOiJFeHRyYSB0ZXh0IGluIG9iamVjdC4iO30=

-

Compression

-

H4sIAAAAAAAA/yu2MjS2UvJIzcnJ11Eozy/KSVFUsgYAZN5upRUAAAA=

- -""" - ) - - def check(self, module_test, events): - for serialize_type in ["Java", "DOTNET", "PHP_Array", "PHP_String", "PHP_Object", "Possible_Compressed"]: - assert any( - e.type == "FINDING" and serialize_type in e.data["description"] for e in events - ), f"Did not find {serialize_type} Serialized Object" - - -class TestExcavateNonHttpScheme(TestExcavate): - - targets = ["http://127.0.0.1:8888/", "test.notreal"] - - non_http_scheme_html = """ - - - - - -

hxxp://test.notreal

-

ftp://test.notreal

-

nonsense://test.notreal

- - - """ - - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data(self.non_http_scheme_html) - - def check(self, module_test, events): - - found_hxxp_url = False - found_ftp_url = False - found_nonsense_url = False - - for e in events: - if e.type == "FINDING": - if e.data["description"] == "Non-HTTP URI: hxxp://test.notreal": - found_hxxp_url = True - if e.data["description"] == "Non-HTTP URI: ftp://test.notreal": - found_ftp_url = True - if "nonsense" in e.data["description"]: - found_nonsense_url = True - assert found_hxxp_url - assert found_ftp_url - assert not found_nonsense_url - - -class TestExcavateParameterExtraction(TestExcavate): - # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER - modules_overrides = ["excavate", "httpx", "hunt"] - targets = ["http://127.0.0.1:8888/"] - parameter_extraction_html = """ - - - Get extract - - - - -

Simple GET Form

-

Use the form below to submit a GET request:

-
- -

- -
-

Simple POST Form

-

Use the form below to submit a POST request:

-
- -

- -
-

Simple Generic Form

-

Use the form below to submit a request:

-
- -

- -
-

Links

- href - img - - - - """ - - async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data(self.parameter_extraction_html) - - def check(self, module_test, events): - found_jquery_get = False - found_jquery_post = False - found_form_get = False - found_form_post = False - found_form_generic = False - found_jquery_get_original_value = False - found_jquery_post_original_value = False - found_form_get_original_value = False - found_form_post_original_value = False - found_form_generic_original_value = False - found_htmltags_a = False - found_htmltags_img = False - - for e in events: - - if e.type == "WEB_PARAMETER": - - if e.data["description"] == "HTTP Extracted Parameter [jqueryget] (GET jquery Submodule)": - found_jquery_get = True - if e.data["original_value"] == "value1": - found_jquery_get_original_value = True - - if e.data["description"] == "HTTP Extracted Parameter [jquerypost] (POST jquery Submodule)": - found_jquery_post = True - if e.data["original_value"] == "value2": - found_jquery_post_original_value = True - - if e.data["description"] == "HTTP Extracted Parameter [q1] (GET Form Submodule)": - found_form_get = True - if e.data["original_value"] == "flowers": - found_form_get_original_value = True - - if e.data["description"] == "HTTP Extracted Parameter [q2] (POST Form Submodule)": - found_form_post = True - if e.data["original_value"] == "boats": - found_form_post_original_value = True - - if e.data["description"] == "HTTP Extracted Parameter [q3] (Generic Form Submodule)": - found_form_generic = True - if e.data["original_value"] == "candles": - found_form_generic_original_value = True - - if e.data["description"] == "HTTP Extracted Parameter [age] (HTML Tags Submodule)": - if e.data["original_value"] == "456": - if "id" in e.data["additional_params"].keys(): - found_htmltags_a = True - - if e.data["description"] == "HTTP Extracted Parameter [size] (HTML Tags Submodule)": - if e.data["original_value"] == "m": - if "fit" in e.data["additional_params"].keys(): - found_htmltags_img = True - - assert found_jquery_get, "Did not extract Jquery GET parameters" - assert found_jquery_post, "Did not extract Jquery POST parameters" - assert found_form_get, "Did not extract Form GET parameters" - assert found_form_post, "Did not extract Form POST parameters" - assert found_form_generic, "Did not extract Form (Generic) parameters" - assert found_jquery_get_original_value, "Did not extract Jquery GET parameter original_value" - assert found_jquery_post_original_value, "Did not extract Jquery POST parameter original_value" - assert found_form_get_original_value, "Did not extract Form GET parameter original_value" - assert found_form_post_original_value, "Did not extract Form POST parameter original_value" - assert found_form_generic_original_value, "Did not extract Form (Generic) parameter original_value" - assert found_htmltags_a, "Did not extract parameter(s) from a-tag" - assert found_htmltags_img, "Did not extract parameter(s) from img-tag" - - -class TestExcavateParameterExtraction_getparam(ModuleTestBase): - - targets = ["http://127.0.0.1:8888/"] - - # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER - modules_overrides = ["httpx", "excavate", "hunt"] - getparam_extract_html = """ -ping - """ - - async def setup_after_prep(self, module_test): - respond_args = {"response_data": self.getparam_extract_html, "headers": {"Content-Type": "text/html"}} - module_test.set_expect_requests(respond_args=respond_args) - - def check(self, module_test, events): - - excavate_getparam_extraction = False - for e in events: - if e.type == "WEB_PARAMETER": - - if "HTTP Extracted Parameter [hack] (HTML Tags Submodule)" in e.data["description"]: - excavate_getparam_extraction = True - assert excavate_getparam_extraction, "Excavate failed to extract web parameter" - - -class TestExcavateParameterExtraction_json(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] - modules_overrides = ["httpx", "excavate", "paramminer_getparams"] - config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}}} - getparam_extract_json = """ - { - "obscureParameter": 1, - "common": 1 -} - """ - - async def setup_after_prep(self, module_test): - respond_args = {"response_data": self.getparam_extract_json, "headers": {"Content-Type": "application/json"}} - module_test.set_expect_requests(respond_args=respond_args) - - def check(self, module_test, events): - excavate_json_extraction = False - for e in events: - if e.type == "WEB_PARAMETER": - if ( - "HTTP Extracted Parameter (speculative from json content) [obscureParameter]" - in e.data["description"] - ): - excavate_json_extraction = True - assert excavate_json_extraction, "Excavate failed to extract json parameter" - - -class TestExcavateParameterExtraction_xml(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] - modules_overrides = ["httpx", "excavate", "paramminer_getparams"] - config_overrides = {"modules": {"paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}}} - getparam_extract_xml = """ - - 1 - 1 - - """ - - async def setup_after_prep(self, module_test): - respond_args = {"response_data": self.getparam_extract_xml, "headers": {"Content-Type": "application/xml"}} - module_test.set_expect_requests(respond_args=respond_args) - - def check(self, module_test, events): - excavate_xml_extraction = False - for e in events: - if e.type == "WEB_PARAMETER": - if ( - "HTTP Extracted Parameter (speculative from xml content) [obscureParameter]" - in e.data["description"] - ): - excavate_xml_extraction = True - assert excavate_xml_extraction, "Excavate failed to extract xml parameter" - - -class excavateTestRule(ExcavateRule): - yara_rules = { - "SearchForText": 'rule SearchForText { meta: description = "Contains the text AAAABBBBCCCC" strings: $text = "AAAABBBBCCCC" condition: $text }', - "SearchForText2": 'rule SearchForText2 { meta: description = "Contains the text DDDDEEEEFFFF" strings: $text2 = "DDDDEEEEFFFF" condition: $text2 }', - } - - -class TestExcavateYara(TestExcavate): - - targets = ["http://127.0.0.1:8888/"] - yara_test_html = """ - - - - -

AAAABBBBCCCC

-

filler

-

DDDDEEEEFFFF

- - -""" - - async def setup_before_prep(self, module_test): - - self.modules_overrides = ["excavate", "httpx"] - module_test.httpserver.expect_request("/").respond_with_data(self.yara_test_html) - - async def setup_after_prep(self, module_test): - - excavate_module = module_test.scan.modules["excavate"] - excavateruleinstance = excavateTestRule(excavate_module) - excavate_module.add_yara_rule( - "SearchForText", - 'rule SearchForText { meta: description = "Contains the text AAAABBBBCCCC" strings: $text = "AAAABBBBCCCC" condition: $text }', - excavateruleinstance, - ) - excavate_module.add_yara_rule( - "SearchForText2", - 'rule SearchForText2 { meta: description = "Contains the text DDDDEEEEFFFF" strings: $text2 = "DDDDEEEEFFFF" condition: $text2 }', - excavateruleinstance, - ) - excavate_module.yara_rules = yara.compile(source="\n".join(excavate_module.yara_rules_dict.values())) - - def check(self, module_test, events): - found_yara_string_1 = False - found_yara_string_2 = False - for e in events: - - if e.type == "FINDING": - if e.data["description"] == "HTTP response (body) Contains the text AAAABBBBCCCC": - found_yara_string_1 = True - if e.data["description"] == "HTTP response (body) Contains the text DDDDEEEEFFFF": - found_yara_string_2 = True - - assert found_yara_string_1, "Did not extract Match YARA rule (1)" - assert found_yara_string_2, "Did not extract Match YARA rule (2)" - - -class TestExcavateYaraCustom(TestExcavateYara): - - rule_file = [ - 'rule SearchForText { meta: description = "Contains the text AAAABBBBCCCC" strings: $text = "AAAABBBBCCCC" condition: $text }', - 'rule SearchForText2 { meta: description = "Contains the text DDDDEEEEFFFF" strings: $text2 = "DDDDEEEEFFFF" condition: $text2 }', - ] - f = tempwordlist(rule_file) - config_overrides = {"modules": {"excavate": {"custom_yara_rules": f}}} - - -class TestExcavateSpiderDedupe(ModuleTestBase): - class DummyModule(BaseModule): - watched_events = ["URL_UNVERIFIED"] - _name = "dummy_module" - - events_seen = [] - - async def handle_event(self, event): - await self.helpers.sleep(0.5) - self.events_seen.append(event.data) - new_event = self.scan.make_event(event.data, "URL_UNVERIFIED", self.scan.root_event) - if new_event is not None: - await self.emit_event(new_event) - - dummy_text = "spider" - modules_overrides = ["excavate", "httpx"] - targets = ["http://127.0.0.1:8888/"] - - async def setup_after_prep(self, module_test): - self.dummy_module = self.DummyModule(module_test.scan) - module_test.scan.modules["dummy_module"] = self.dummy_module - module_test.httpserver.expect_request("/").respond_with_data(self.dummy_text) - module_test.httpserver.expect_request("/spider").respond_with_data("hi") - - def check(self, module_test, events): - - found_url_unverified_spider_max = False - found_url_unverified_dummy = False - found_url_event = False - - assert sorted(self.dummy_module.events_seen) == ["http://127.0.0.1:8888/", "http://127.0.0.1:8888/spider"] - - for e in events: - if e.type == "URL_UNVERIFIED": - if e.data == "http://127.0.0.1:8888/spider": - if str(e.module) == "excavate" and "spider-danger" in e.tags and "spider-max" in e.tags: - found_url_unverified_spider_max = True - if ( - str(e.module) == "dummy_module" - and "spider-danger" not in e.tags - and not "spider-max" in e.tags - ): - found_url_unverified_dummy = True - if e.type == "URL" and e.data == "http://127.0.0.1:8888/spider": - found_url_event = True - - assert found_url_unverified_spider_max, "Excavate failed to find /spider link" - assert found_url_unverified_dummy, "Dummy module did not correctly re-emit" - assert found_url_event, "URL was not emitted from non-spider-max URL_UNVERIFIED" - - -class TestExcavateParameterExtraction_targeturl(ModuleTestBase): - targets = ["http://127.0.0.1:8888/?foo=1"] - modules_overrides = ["httpx", "excavate", "hunt"] - config_overrides = { - "url_querystring_remove": False, - "url_querystring_collapse": False, - "interactsh_disable": True, - "modules": { - "excavate": { - "retain_querystring": True, - } - }, - } - - async def setup_after_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/", "query_string": "foo=1"} - respond_args = { - "response_data": "alive", - "status": 200, - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check(self, module_test, events): - web_parameter_emit = False - for e in events: - if e.type == "WEB_PARAMETER" and "HTTP Extracted Parameter [foo] (Target URL)" in e.data["description"]: - web_parameter_emit = True - - assert web_parameter_emit - - -class TestExcavate_retain_querystring(ModuleTestBase): - targets = ["http://127.0.0.1:8888/?foo=1"] - modules_overrides = ["httpx", "excavate", "hunt"] - config_overrides = { - "url_querystring_remove": False, - "url_querystring_collapse": False, - "interactsh_disable": True, - "web_spider_depth": 4, - "web_spider_distance": 4, - "modules": { - "excavate": { - "retain_querystring": True, - } - }, - } - - async def setup_after_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/", "query_string": "foo=1"} - respond_args = { - "response_data": "alive", - "headers": {"Set-Cookie": "a=b"}, - "status": 200, - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check(self, module_test, events): - web_parameter_emit = False - for e in events: - if e.type == "WEB_PARAMETER" and "foo" in e.data["url"]: - web_parameter_emit = True - - assert web_parameter_emit - - -class TestExcavate_retain_querystring_not(TestExcavate_retain_querystring): - - config_overrides = { - "url_querystring_remove": False, - "url_querystring_collapse": False, - "interactsh_disable": True, - "web_spider_depth": 4, - "web_spider_distance": 4, - "modules": { - "excavate": { - "retain_querystring": True, - } - }, - } - - def check(self, module_test, events): - web_parameter_emit = False - for e in events: - if e.type == "WEB_PARAMETER" and "foo" not in e.data["url"]: - web_parameter_emit = True - - assert web_parameter_emit - - -class TestExcavate_webparameter_outofscope(ModuleTestBase): - - html_body = "" - - targets = ["http://127.0.0.1:8888", "socialmediasite.com"] - modules_overrides = ["httpx", "excavate", "hunt"] - config_overrides = {"interactsh_disable": True} - - async def setup_after_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/"} - respond_args = { - "response_data": self.html_body, - "status": 200, - } - module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) - - def check(self, module_test, events): - web_parameter_differentsite = False - web_parameter_outofscope = False - - for e in events: - if e.type == "WEB_PARAMETER" and "in-scope" in e.tags and e.host == "socialmediasite.com": - web_parameter_differentsite = True - - if e.type == "WEB_PARAMETER" and e.host == "outofscope.com": - web_parameter_outofscope = True - - assert web_parameter_differentsite, "WEB_PARAMETER was not emitted" - assert not web_parameter_outofscope, "Out of scope domain was emitted" - - -class TestExcavateHeaders(ModuleTestBase): - - targets = ["http://127.0.0.1:8888/"] - modules_overrides = ["excavate", "httpx", "hunt"] - config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} - - async def setup_before_prep(self, module_test): - - module_test.httpserver.expect_request("/").respond_with_data( - "

test

", - status=200, - headers={ - "Set-Cookie": [ - "COOKIE1=aaaa; Secure; HttpOnly", - "COOKIE2=bbbb; Secure; HttpOnly; SameSite=None", - ] - }, - ) - - def check(self, module_test, events): - - found_first_cookie = False - found_second_cookie = False - - for e in events: - if e.type == "WEB_PARAMETER": - if e.data["name"] == "COOKIE1": - found_first_cookie = True - if e.data["name"] == "COOKIE2": - found_second_cookie = True - - assert found_first_cookie == True - assert found_second_cookie == True - - class TestExcavateHeaders_blacklist(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 36ac57f29..4c5472130 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -430,8 +430,6 @@ class Test_Lightfuzz_sqli_post(ModuleTestBase): def request_handler(self, request): - qs = str(request.query_string.decode()) - parameter_block = """
@@ -558,7 +556,6 @@ def request_handler(self, request): """ - qs = str(request.query_string.decode()) if request.headers.get("Test") is not None: header_value = request.headers.get("Test") @@ -580,8 +577,6 @@ def request_handler(self, request): return Response(placeholder_block, status=200) def check(self, module_test, events): - - web_parameter_emitted = False sqli_finding_emitted = False for e in events: if e.type == "FINDING": @@ -631,7 +626,6 @@ def request_handler(self, request): """ - qs = str(request.query_string.decode()) if request.cookies.get("test") is not None: header_value = request.cookies.get("test") @@ -655,7 +649,6 @@ def request_handler(self, request): def check(self, module_test, events): - web_parameter_emitted = False sqli_finding_emitted = False for e in events: if e.type == "FINDING": @@ -806,7 +799,6 @@ def check(self, module_test, events): excavate_extracted_form_parameter = False excavate_extracted_form_parameter_details = False lightfuzz_serial_detect_errorresolution = False - lightfuzz_serial_detect_errordifferential = False for e in events: if e.type == "WEB_PARAMETER": From 2ad217730e76e3e9d003c94fd7963349d29e70bc Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 11:11:51 -0400 Subject: [PATCH 147/238] dependency update --- poetry.lock | 132 +++++++++++++++++++++++++++++++------------------ pyproject.toml | 1 + 2 files changed, 86 insertions(+), 47 deletions(-) diff --git a/poetry.lock b/poetry.lock index 75b27e134..3094753ce 100644 --- a/poetry.lock +++ b/poetry.lock @@ -728,13 +728,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.5" +version = "1.0.6" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, - {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, + {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, + {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, ] [package.dependencies] @@ -745,7 +745,7 @@ h11 = ">=0.13,<0.15" asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<0.26.0)"] +trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httpx" @@ -1676,6 +1676,44 @@ files = [ {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, ] +[[package]] +name = "pyahocorasick" +version = "2.1.0" +description = "pyahocorasick is a fast and memory efficient library for exact or approximate multi-pattern string search. With the ``ahocorasick.Automaton`` class, you can find multiple key string occurrences at once in some input text. You can use it as a plain dict-like Trie or convert a Trie to an automaton for efficient Aho-Corasick search. And pickle to disk for easy reuse of large automatons. Implemented in C and tested on Python 3.6+. Works on Linux, macOS and Windows. BSD-3-Cause license." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyahocorasick-2.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c46288044c4f71392efb4f5da0cb8abd160787a8b027afc85079e9c3d7551eb"}, + {file = "pyahocorasick-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f15529c83b8c6e0548d7d3c5631fefa23fba5190e67be49d6c9e24a6358ff9c"}, + {file = "pyahocorasick-2.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:581e3d85043f1797543796f021e8d7d48c18e594529b72d86f70ea78abc88fff"}, + {file = "pyahocorasick-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c860ad9cb59e56c31aed8a5d1ee9d83a0151277b09198d027ffce213697716ed"}, + {file = "pyahocorasick-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4f8eba88fce34a1d8020638a4a8732c6241a5d85fe12be8669b7495d99d36b6a"}, + {file = "pyahocorasick-2.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d6e0da0a8fc78c694778dced537c1bfb8b2f178ec92a82d81539d2e35a15cba0"}, + {file = "pyahocorasick-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:658d55e51c7588a5dba57de674241a16a3c94bf57f3bfd70022c4d7defe2b0f4"}, + {file = "pyahocorasick-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9f2728ac77bab807ba65c6ef41be30358ef0c9bb6960c9fe070d43f7024cb91"}, + {file = "pyahocorasick-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a58c44c407a45155dc7a3253274b5fd78ab00b579bd5685059610867cdb37142"}, + {file = "pyahocorasick-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d8254d6333df5eb400ed3ec8b24da9e3f5da8e28b94a71392391703a7aac568d"}, + {file = "pyahocorasick-2.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:82b0d20e82cc282fd29324e8df93809cebbffb345055214ce4b7873698df02c8"}, + {file = "pyahocorasick-2.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6dedb9fed92705b742d6aa3d87abb1ec999f57310ef32b962f65f4e42182fe0a"}, + {file = "pyahocorasick-2.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f209796e7d354734781dd883c333596e482c70136fa76a4cb169f383e6c40bca"}, + {file = "pyahocorasick-2.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8337af64c649223cff548c7204dda823e83622d63e5449bc51ae069efb2f240f"}, + {file = "pyahocorasick-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:5ebe0d1e15afb782477e3d0aa1dce28ab9dad1200211fb785b9c1cc1208e6f04"}, + {file = "pyahocorasick-2.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7454ba5fa528958ca9a1bc3143f8e980bd7817ea481f46495e6ffa89675ab93b"}, + {file = "pyahocorasick-2.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3795ac922d21fbfea40a6b3a330762e8b38ce8ba511b1eb15bf9eeb9303b2662"}, + {file = "pyahocorasick-2.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8e92150849a3c13da37e37ca6374fa55960fd5c845029eca02d9b5846b26fe48"}, + {file = "pyahocorasick-2.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:23b183600e2087f16f6c5e6185d61525ad74335f2a5b693dd6d66bba2f6a4b05"}, + {file = "pyahocorasick-2.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:7034b26e145518610651339b8701568a3533a3114b00cf55f22bca80bff58e6d"}, + {file = "pyahocorasick-2.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:36491675a13fe4181a6b3bccfc9032a1a5d03bd3b0a151c06f8865c16ba44b42"}, + {file = "pyahocorasick-2.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:895ab1ff5384ee5325c74cbacafc419e534f1f110b9fb3c544cc56832ecce082"}, + {file = "pyahocorasick-2.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:bf4a4b19ac37e9a7087646b8bcc306acd7a91649355d59b866b756068e35d018"}, + {file = "pyahocorasick-2.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f44f96496aa773fc5bf302ddf968dd6b920fab34522f944392af8bde13cbe805"}, + {file = "pyahocorasick-2.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:05b7c2ef52da247efec6fb5a011113b7e943e961e22aaaf757cb9c15083440c9"}, + {file = "pyahocorasick-2.1.0.tar.gz", hash = "sha256:4df4845c1149e9fa4aa33f0f0aa35f5a42957a43a3d6e447c9b44e679e2672ea"}, +] + +[package.extras] +testing = ["pytest", "setuptools", "twine", "wheel"] + [[package]] name = "pycodestyle" version = "2.12.1" @@ -1700,43 +1738,43 @@ files = [ [[package]] name = "pycryptodome" -version = "3.20.0" +version = "3.21.0" description = "Cryptographic library for Python" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "pycryptodome-3.20.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:f0e6d631bae3f231d3634f91ae4da7a960f7ff87f2865b2d2b831af1dfb04e9a"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:baee115a9ba6c5d2709a1e88ffe62b73ecc044852a925dcb67713a288c4ec70f"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:417a276aaa9cb3be91f9014e9d18d10e840a7a9b9a9be64a42f553c5b50b4d1d"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1250b7ea809f752b68e3e6f3fd946b5939a52eaeea18c73bdab53e9ba3c2dd"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:d5954acfe9e00bc83ed9f5cb082ed22c592fbbef86dc48b907238be64ead5c33"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-win32.whl", hash = "sha256:06d6de87c19f967f03b4cf9b34e538ef46e99a337e9a61a77dbe44b2cbcf0690"}, - {file = "pycryptodome-3.20.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ec0bb1188c1d13426039af8ffcb4dbe3aad1d7680c35a62d8eaf2a529b5d3d4f"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5601c934c498cd267640b57569e73793cb9a83506f7c73a8ec57a516f5b0b091"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d29daa681517f4bc318cd8a23af87e1f2a7bad2fe361e8aa29c77d652a065de4"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3427d9e5310af6680678f4cce149f54e0bb4af60101c7f2c16fdf878b39ccccc"}, - {file = "pycryptodome-3.20.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:3cd3ef3aee1079ae44afaeee13393cf68b1058f70576b11439483e34f93cf818"}, - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac1c7c0624a862f2e53438a15c9259d1655325fc2ec4392e66dc46cdae24d044"}, - {file = "pycryptodome-3.20.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:76658f0d942051d12a9bd08ca1b6b34fd762a8ee4240984f7c06ddfb55eaf15a"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f35d6cee81fa145333137009d9c8ba90951d7d77b67c79cbe5f03c7eb74d8fe2"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76cb39afede7055127e35a444c1c041d2e8d2f1f9c121ecef573757ba4cd2c3c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49a4c4dc60b78ec41d2afa392491d788c2e06edf48580fbfb0dd0f828af49d25"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fb3b87461fa35afa19c971b0a2b7456a7b1db7b4eba9a8424666104925b78128"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:acc2614e2e5346a4a4eab6e199203034924313626f9620b7b4b38e9ad74b7e0c"}, - {file = "pycryptodome-3.20.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:210ba1b647837bfc42dd5a813cdecb5b86193ae11a3f5d972b9a0ae2c7e9e4b4"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win32.whl", hash = "sha256:8d6b98d0d83d21fb757a182d52940d028564efe8147baa9ce0f38d057104ae72"}, - {file = "pycryptodome-3.20.0-cp35-abi3-win_amd64.whl", hash = "sha256:9b3ae153c89a480a0ec402e23db8d8d84a3833b65fa4b15b81b83be9d637aab9"}, - {file = "pycryptodome-3.20.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:4401564ebf37dfde45d096974c7a159b52eeabd9969135f0426907db367a652a"}, - {file = "pycryptodome-3.20.0-pp27-pypy_73-win32.whl", hash = "sha256:ec1f93feb3bb93380ab0ebf8b859e8e5678c0f010d2d78367cf6bc30bfeb148e"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:acae12b9ede49f38eb0ef76fdec2df2e94aad85ae46ec85be3648a57f0a7db04"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f47888542a0633baff535a04726948e876bf1ed880fddb7c10a736fa99146ab3"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e0e4a987d38cfc2e71b4a1b591bae4891eeabe5fa0f56154f576e26287bfdea"}, - {file = "pycryptodome-3.20.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c18b381553638414b38705f07d1ef0a7cf301bc78a5f9bc17a957eb19446834b"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a60fedd2b37b4cb11ccb5d0399efe26db9e0dd149016c1cc6c8161974ceac2d6"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:405002eafad114a2f9a930f5db65feef7b53c4784495dd8758069b89baf68eab"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab6ab0cb755154ad14e507d1df72de9897e99fd2d4922851a276ccc14f4f1a5"}, - {file = "pycryptodome-3.20.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:acf6e43fa75aca2d33e93409f2dafe386fe051818ee79ee8a3e21de9caa2ac9e"}, - {file = "pycryptodome-3.20.0.tar.gz", hash = "sha256:09609209ed7de61c2b560cc5c8c4fbf892f8b15b1faf7e4cbffac97db1fffda7"}, +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pycryptodome-3.21.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dad9bf36eda068e89059d1f07408e397856be9511d7113ea4b586642a429a4fd"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:a1752eca64c60852f38bb29e2c86fca30d7672c024128ef5d70cc15868fa10f4"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:3ba4cc304eac4d4d458f508d4955a88ba25026890e8abff9b60404f76a62c55e"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cb087b8612c8a1a14cf37dd754685be9a8d9869bed2ffaaceb04850a8aeef7e"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:26412b21df30b2861424a6c6d5b1d8ca8107612a4cfa4d0183e71c5d200fb34a"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-win32.whl", hash = "sha256:cc2269ab4bce40b027b49663d61d816903a4bd90ad88cb99ed561aadb3888dd3"}, + {file = "pycryptodome-3.21.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0fa0a05a6a697ccbf2a12cec3d6d2650b50881899b845fac6e87416f8cb7e87d"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:6cce52e196a5f1d6797ff7946cdff2038d3b5f0aba4a43cb6bf46b575fd1b5bb"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:a915597ffccabe902e7090e199a7bf7a381c5506a747d5e9d27ba55197a2c568"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4e74c522d630766b03a836c15bff77cb657c5fdf098abf8b1ada2aebc7d0819"}, + {file = "pycryptodome-3.21.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:a3804675283f4764a02db05f5191eb8fec2bb6ca34d466167fc78a5f05bbe6b3"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:2480ec2c72438430da9f601ebc12c518c093c13111a5c1644c82cdfc2e50b1e4"}, + {file = "pycryptodome-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:de18954104667f565e2fbb4783b56667f30fb49c4d79b346f52a29cb198d5b6b"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de4b7263a33947ff440412339cb72b28a5a4c769b5c1ca19e33dd6cd1dcec6e"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0714206d467fc911042d01ea3a1847c847bc10884cf674c82e12915cfe1649f8"}, + {file = "pycryptodome-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d85c1b613121ed3dbaa5a97369b3b757909531a959d229406a75b912dd51dd1"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8898a66425a57bcf15e25fc19c12490b87bd939800f39a03ea2de2aea5e3611a"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:932c905b71a56474bff8a9c014030bc3c882cee696b448af920399f730a650c2"}, + {file = "pycryptodome-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:18caa8cfbc676eaaf28613637a89980ad2fd96e00c564135bf90bc3f0b34dd93"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win32.whl", hash = "sha256:280b67d20e33bb63171d55b1067f61fbd932e0b1ad976b3a184303a3dad22764"}, + {file = "pycryptodome-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b7aa25fc0baa5b1d95b7633af4f5f1838467f1815442b22487426f94e0d66c53"}, + {file = "pycryptodome-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:2cb635b67011bc147c257e61ce864879ffe6d03342dc74b6045059dfbdedafca"}, + {file = "pycryptodome-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:4c26a2f0dc15f81ea3afa3b0c87b87e501f235d332b7f27e2225ecb80c0b1cdd"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d5ebe0763c982f069d3877832254f64974139f4f9655058452603ff559c482e8"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee86cbde706be13f2dec5a42b52b1c1d1cbb90c8e405c68d0755134735c8dc6"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0fd54003ec3ce4e0f16c484a10bc5d8b9bd77fa662a12b85779a2d2d85d67ee0"}, + {file = "pycryptodome-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5dfafca172933506773482b0e18f0cd766fd3920bd03ec85a283df90d8a17bc6"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:590ef0898a4b0a15485b05210b4a1c9de8806d3ad3d47f74ab1dc07c67a6827f"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f35e442630bc4bc2e1878482d6f59ea22e280d7121d7adeaedba58c23ab6386b"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff99f952db3db2fbe98a0b355175f93ec334ba3d01bbde25ad3a5a33abc02b58"}, + {file = "pycryptodome-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8acd7d34af70ee63f9a849f957558e49a98f8f1634f86a59d2be62bb8e93f71c"}, + {file = "pycryptodome-3.21.0.tar.gz", hash = "sha256:f7787e0d469bdae763b876174cf2e6c0f7be79808af26b1da96f1a64bcf47297"}, ] [[package]] @@ -1907,13 +1945,13 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pymdown-extensions" -version = "10.11.1" +version = "10.11.2" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.8" files = [ - {file = "pymdown_extensions-10.11.1-py3-none-any.whl", hash = "sha256:a2b28f5786e041f19cb5bb30a1c2c853668a7099da8e3dd822a5ad05f2e855e3"}, - {file = "pymdown_extensions-10.11.1.tar.gz", hash = "sha256:a8836e955851542fa2625d04d59fdf97125ca001377478ed5618e04f9183a59a"}, + {file = "pymdown_extensions-10.11.2-py3-none-any.whl", hash = "sha256:41cdde0a77290e480cf53892f5c5e50921a7ee3e5cd60ba91bf19837b33badcf"}, + {file = "pymdown_extensions-10.11.2.tar.gz", hash = "sha256:bc8847ecc9e784a098efd35e20cba772bc5a1b529dfcef9dc1972db9021a1049"}, ] [package.dependencies] @@ -2672,13 +2710,13 @@ testing = ["black", "mypy", "pytest", "pytest-gitignore", "pytest-mock", "respon [[package]] name = "tomli" -version = "2.0.1" +version = "2.0.2" description = "A lil' TOML parser" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] [[package]] @@ -3071,4 +3109,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "fb2f904f634456d6e72915cdf3040249aa2dc498106b1879732e0276df7ea082" +content-hash = "1bd4ecfe101d19e182401daf146486d6bcba963b802de78d8f8955454eb992b5" diff --git a/pyproject.toml b/pyproject.toml index b0bbf483b..0b8f1580d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ setproctitle = "^1.3.3" yara-python = "^4.5.1" pyzmq = "^26.0.3" httpx = "^0.27.0" +pyahocorasick = "^2.1.0" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" From ecee809be3cfcd9b22e7f2b05d0f9846b0b44875 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 11:21:33 -0400 Subject: [PATCH 148/238] fixing module pre-loading logic --- bbot/core/modules.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 7fd38a33f..682994166 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -104,8 +104,9 @@ def add_module_dir(self, module_dir): def file_filter(self, file): file = file.resolve() - if "templates" in file.parts: - return False + for part in file.parts: + if part.endswith("_submodules") or part == "templates": + return False return file.suffix.lower() == ".py" and file.stem not in ["base", "__init__"] def preload(self, module_dirs=None): From 01e7a26d63c8ba4c5445d1150169cb03472b6d75 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 11:40:08 -0400 Subject: [PATCH 149/238] flag compliance --- bbot/modules/lightfuzz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 8136ee2d9..1d14cf425 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -15,7 +15,7 @@ class lightfuzz(BaseModule): watched_events = ["URL", "WEB_PARAMETER"] produced_events = ["FINDING", "VULNERABILITY"] - flags = ["active", "web-thorough"] + flags = ["active", "aggressive", "web-thorough"] submodules = { "sqli": {"description": "SQL Injection", "module": SQLiLightfuzz}, From a0081f0b5ac3d93b8159c79f5de03aff6d42e132 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 12:06:06 -0400 Subject: [PATCH 150/238] adding author --- bbot/modules/lightfuzz.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 1d14cf425..1f724eb9a 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -34,7 +34,11 @@ class lightfuzz(BaseModule): "disable_post": "Disable processing of POST parameters, avoiding form submissions.", } - meta = {"description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner"} + meta = { + "description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner", + "author": "@liquidsec", + "created_date": "2024-06-28", + } common_headers = ["x-forwarded-for", "user-agent"] in_scope_only = True From cda0bd7ecc6acbc25a185ee7d8a4f111c9105dd6 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 12:31:44 -0400 Subject: [PATCH 151/238] add cookies cli bug fix --- bbot/core/helpers/web/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/web/web.py b/bbot/core/helpers/web/web.py index 5220df7aa..a6bbdff76 100644 --- a/bbot/core/helpers/web/web.py +++ b/bbot/core/helpers/web/web.py @@ -354,7 +354,7 @@ async def curl(self, *args, **kwargs): if hk not in headers: headers[hk] = hv - for ck, cv in self._web_config.get("http_cookies", {}).items(): + for ck, cv in self.web_config.get("http_cookies", {}).items(): # don't clobber cookies if ck not in cookies: cookies[ck] = cv From 8ff836beb650dc55722289528ad0cb89eea87017 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 13:20:01 -0400 Subject: [PATCH 152/238] fixing bug with rand_string helper --- bbot/core/helpers/misc.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index e9f7375f7..a1102c81c 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -792,10 +792,6 @@ def recursive_decode(data, max_depth=5): return data -rand_pool = string.ascii_lowercase -rand_pool_digits = rand_pool + string.digits - - def rand_string(length=10, digits=True, numeric_only=False): """ Generates a random string of specified length. @@ -821,9 +817,9 @@ def rand_string(length=10, digits=True, numeric_only=False): if numeric_only: pool = string.digits elif digits: - pool = string.ascii_letters + string.digits + pool = string.ascii_lowercase + string.digits else: - pool = string.ascii_letters + pool = string.ascii_lowercase return "".join(random.choice(pool) for _ in range(length)) From c0f9e43916afb98388137e06171b312132ce13a6 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 3 Oct 2024 21:22:09 -0400 Subject: [PATCH 153/238] always add original value --- bbot/modules/paramminer_headers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index ca2894ce3..4919c3c98 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -149,6 +149,7 @@ async def process_results(self, event, results): "type": paramtype, "description": description, "name": result, + "original_value": None, }, "WEB_PARAMETER", event, From c70db9b7a8abe57a91f6eea5a10c6d797a66596a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 10:21:26 -0400 Subject: [PATCH 154/238] handling failure to get all test responses sqli --- bbot/modules/lightfuzz_submodules/sqli.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 664fca92b..0178f31d0 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -57,13 +57,16 @@ async def fuzz(self): additional_params_populate_empty=True, ) - if "code" in single_quote[1] and (single_quote[3].status_code != double_single_quote[3].status_code): - self.results.append( - { - "type": "FINDING", - "description": f"Possible SQL Injection. {self.metadata()} Detection Method: [Single Quote/Two Single Quote]", - } - ) + if single_quote and double_single_quote: + if "code" in single_quote[1] and (single_quote[3].status_code != double_single_quote[3].status_code): + self.results.append( + { + "type": "FINDING", + "description": f"Possible SQL Injection. {self.metadata()} Detection Method: [Single Quote/Two Single Quote]", + } + ) + else: + self.lightfuzz.debug("Failed to get responses for both single_quote and double_single_quote") except HttpCompareError as e: self.lightfuzz.warning(f"Encountered HttpCompareError Sending Compare Probe: {e}") From 68c934b0fc0227d3c97990f1696f24844b01ac78 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 11:38:05 -0400 Subject: [PATCH 155/238] handling failure to get all test responses sqli (again) --- bbot/modules/lightfuzz_submodules/sqli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 0178f31d0..9d5db9518 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -57,7 +57,7 @@ async def fuzz(self): additional_params_populate_empty=True, ) - if single_quote and double_single_quote: + if single_quote[3] and double_single_quote[3]: if "code" in single_quote[1] and (single_quote[3].status_code != double_single_quote[3].status_code): self.results.append( { From 0416feafffed8911cbd4d1c40bdd5c3c1be148ac Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 13:26:40 -0400 Subject: [PATCH 156/238] fixing preset web spider settings --- bbot/presets/web/lightfuzz-intense.yml | 5 +++-- bbot/presets/web/lightfuzz-max.yml | 5 +++-- bbot/presets/web/lightfuzz-xss.yml | 5 +++-- bbot/presets/web/lightfuzz.yml | 5 +++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml index 22fa02fc8..16caecbca 100644 --- a/bbot/presets/web/lightfuzz-intense.yml +++ b/bbot/presets/web/lightfuzz-intense.yml @@ -13,8 +13,9 @@ modules: config: url_querystring_remove: False url_querystring_collapse: True - web_spider_distance: 4 - web_spider_depth: 5 + web: + spider_distance: 4 + spider_depth: 5 modules: lightfuzz: force_common_headers: False diff --git a/bbot/presets/web/lightfuzz-max.yml b/bbot/presets/web/lightfuzz-max.yml index c0ede2fbb..e0c7a738b 100644 --- a/bbot/presets/web/lightfuzz-max.yml +++ b/bbot/presets/web/lightfuzz-max.yml @@ -13,8 +13,9 @@ modules: config: url_querystring_remove: False url_querystring_collapse: False - web_spider_distance: 4 - web_spider_depth: 5 + web: + spider_distance: 4 + spider_depth: 5 modules: lightfuzz: force_common_headers: True diff --git a/bbot/presets/web/lightfuzz-xss.yml b/bbot/presets/web/lightfuzz-xss.yml index 5fcb3117e..fd13b2c67 100644 --- a/bbot/presets/web/lightfuzz-xss.yml +++ b/bbot/presets/web/lightfuzz-xss.yml @@ -7,8 +7,9 @@ modules: config: url_querystring_remove: False url_querystring_collapse: False - web_spider_distance: 4 - web_spider_depth: 5 + web: + spider_distance: 4 + spider_depth: 5 modules: lightfuzz: submodules_enabled: [xss] \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml index 8466d4a8e..85cce074b 100644 --- a/bbot/presets/web/lightfuzz.yml +++ b/bbot/presets/web/lightfuzz.yml @@ -8,8 +8,9 @@ modules: config: url_querystring_remove: False - web_spider_distance: 3 - web_spider_depth: 4 + web: + spider_distance: 3 + spider_depth: 4 modules: lightfuzz: submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] From 5ab40ebd488f2048a3dbcd225cbe183937ef93e3 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 4 Oct 2024 15:40:23 -0400 Subject: [PATCH 157/238] add escapable slash javascript xss detection --- bbot/modules/lightfuzz_submodules/xss.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 326790e9b..0f5b71c07 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -81,4 +81,12 @@ async def fuzz(self): if in_javascript: in_javascript_probe = rf"" - await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") + result = await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") + if result == False: + in_javasscript_escape_probe = rf"a\';zzzzz({random_string})\\" + in_javasscript_escape_match = rf"a\\';zzzzz({random_string})\\" + await self.check_probe( + in_javasscript_escape_probe, + in_javasscript_escape_match, + "In Javascript (escaping the escape character)", + ) From 5d8e0d379262bd93c2c59d04db5185c4e6df8de8 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:16:04 -0400 Subject: [PATCH 158/238] lightfuzz envelope system --- bbot/core/event/base.py | 407 +++++++++++++++++- bbot/core/helpers/misc.py | 2 + bbot/core/helpers/regexes.py | 6 +- bbot/modules/internal/excavate.py | 44 +- bbot/modules/lightfuzz.py | 10 + bbot/modules/lightfuzz_submodules/base.py | 53 ++- bbot/modules/lightfuzz_submodules/cmdi.py | 3 +- bbot/modules/lightfuzz_submodules/crypto.py | 31 +- bbot/modules/lightfuzz_submodules/path.py | 2 +- bbot/modules/lightfuzz_submodules/serial.py | 3 +- bbot/modules/lightfuzz_submodules/sqli.py | 9 +- bbot/presets/web/lightfuzz-intense.yml | 2 +- bbot/presets/web/lightfuzz-max.yml | 2 +- bbot/presets/web/lightfuzz-xss.yml | 2 +- bbot/presets/web/lightfuzz.yml | 2 +- .../module_tests/test_module_excavate.py | 23 + .../module_tests/test_module_lightfuzz.py | 317 ++++++++++++++ 17 files changed, 860 insertions(+), 58 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index d5a9552e2..b096aa095 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,20 +1,26 @@ +from collections import defaultdict import io import re +import threading import uuid import json import base64 +import inspect +import asyncio import logging import tarfile +import binascii import datetime import ipaddress import traceback +import xml.etree.ElementTree as ET -from copy import copy +from copy import copy, deepcopy from pathlib import Path from typing import Optional from contextlib import suppress from radixtarget import RadixTarget -from urllib.parse import urljoin, parse_qs +from urllib.parse import urljoin, parse_qs, unquote, quote from pydantic import BaseModel, field_validator @@ -618,6 +624,23 @@ def get_parents(self, omit=False, include_self=False): e = parent return parents + def clone(self): + + # Create a shallow copy of the event first + cloned_event = copy(self) + + # Handle attributes that need deep copying manually + setattr(cloned_event, "envelopes", deepcopy(self.envelopes)) + # cloned_event.scan = deepcopy(self.scan) + + # Re-assign a new UUID + cloned_event.uuid = uuid.uuid4() + + # Re-create the unpickleable lock object + cloned_event.lock = threading.RLock() + + return cloned_event + def _host(self): return "" @@ -797,7 +820,13 @@ def json(self, mode="json", siem_friendly=False): j["discovery_path"] = self.discovery_path j["parent_chain"] = self.parent_chain + # parameter envelopes + parameter_envelopes = getattr(self, "envelopes", None) + if parameter_envelopes is not None: + j["envelopes"] = parameter_envelopes.to_dict() + # normalize non-primitive python objects + for k, v in list(j.items()): if k == "data": continue @@ -1281,12 +1310,382 @@ class URL_HINT(URL_UNVERIFIED): class WEB_PARAMETER(DictHostEvent): + class ParameterEnvelopes: + + @staticmethod + def preprocess_base64(base64_str): + return base64.b64decode(base64_str).decode() + + @staticmethod + def postprocess_base64(string): + return base64.b64encode(string.encode()).decode() + + @staticmethod + def preprocess_hex(hex_str): + return bytes.fromhex(hex_str).decode() + + @staticmethod + def postprocess_hex(string): + return string.encode().hex() + + @staticmethod + def preprocess_urlencoded(url_encoded_str): + return unquote(url_encoded_str) + + @staticmethod + def postprocess_urlencoded(string): + return quote(string) + + @staticmethod + def is_ascii_printable(s): + return all(32 <= ord(char) < 127 for char in s) + + # Converts XML ElementTree to a JSON-like dictionary + def xml_to_dict(self, elem): + """ + Convert XML ElementTree to a dictionary recursively. + """ + d = {elem.tag: {} if elem.attrib else None} + children = list(elem) + if children: + dd = defaultdict(list) + for dc in map(self.xml_to_dict, children): + for k, v in dc.items(): + dd[k].append(v) + d = {elem.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}} + if elem.attrib: + d[elem.tag].update(("@" + k, v) for k, v in elem.attrib.items()) + if elem.text: + text = elem.text.strip() + if children or elem.attrib: + if text: + d[elem.tag]["#text"] = text + else: + d[elem.tag] = text + return d + + def dict_to_xml(self, d): + """ + Converts a dictionary to an XML string without adding an extra root node. + Assumes the dictionary was originally an XML structure. + """ + if not isinstance(d, dict) or len(d) != 1: + raise ValueError("Expected a dictionary with a single root element.") + + # Get the root element directly from the dict keys + root_tag = list(d.keys())[0] + root_element = ET.Element(root_tag) + + # Recursive function to handle nested dicts + def _build_tree(element, subdict): + for key, value in subdict.items(): + if isinstance(value, dict): + # Nested element + sub_element = ET.SubElement(element, key) + _build_tree(sub_element, value) + else: + # Leaf element + sub_element = ET.SubElement(element, key) + sub_element.text = str(value) + + # Start building the tree + _build_tree(root_element, d[root_tag]) + + return ET.tostring(root_element, encoding="utf-8").decode("utf-8") + + @staticmethod + def is_ascii_printable(s): + return all(32 <= ord(char) < 127 for char in s) + + preprocess_map = { + "base64": preprocess_base64, + "hex": preprocess_hex, + "url-encoded": preprocess_urlencoded, + } + postprocess_map = { + "base64": postprocess_base64, + "hex": postprocess_hex, + "url-encoded": postprocess_urlencoded, + } + + # Format-specific functions for isolating and updating parameters + format_isolate_map = { + "json": lambda self: self.isolate_parameter(), + "xml": lambda self: self.isolate_parameter(), + } + format_update_map = { + "json": lambda self, value: self.update_json_parameter(value), + "xml": lambda self, value: self.update_xml_parameter(value), # Placeholder + } + + def initialize_value(self, value=None): + self.envelopes, end_format_dict = self.recurse_envelopes(value) + if self.envelopes: + log.debug(f"Discovered the following envelopes: [{','.join(self.envelopes)}]") + + if end_format_dict is not None: + self.end_format_type = list(end_format_dict.keys())[0] + log.debug(f"Identified the following end format: [{self.end_format_type}]") + self.end_format_data = list(end_format_dict.values())[0] + else: + self.end_format_type = None + self.end_format_data = None + self.end_format_subparameter = None + + def remove_envelopes(self, value): + """ + Remove envelopes from the value, processing each envelope in the order it was applied. + If the final format is present, trigger the appropriate handler (e.g., for JSON). + """ + # Apply the preprocess functions in the order the envelopes were applied + for env in self.envelopes: + func = self.preprocess_map.get(env) + if func: + value = func(value) + + # Dynamically select the appropriate isolate function based on the final format + isolate_func = self.format_isolate_map.get(self.end_format_type) + if isolate_func: + return isolate_func(self) + + return value + + def add_envelopes(self, value): + """ + Add envelopes back to the value, processing in reverse order. + If the final format is present, trigger the appropriate handler (e.g., for JSON). + """ + # Dynamically select the appropriate update function based on the final format + update_func = self.format_update_map.get(self.end_format_type) + if update_func: + value = update_func(self, value) + + # Apply the envelopes in reverse order + for env in self.envelopes[::-1]: + func = self.postprocess_map.get(env) + if func: + value = func(value) + return value + + def recurse_envelopes(self, value, envelopes=None, end_format=None): + if envelopes is None: + envelopes = [] + log.debug( + f"Starting recurse with value: {value}, current envelopes: {', '.join(envelopes)}, current end format: {end_format}" + ) + + if value is None or value == "" or isinstance(value, int): + return envelopes, end_format + + # Try URL decoding + try: + decoded_url = unquote(value) + if decoded_url != value and self.is_ascii_printable(decoded_url): + envelopes.append("url-encoded") + envelopes, end_format_dict = self.recurse_envelopes(decoded_url, envelopes) + return envelopes, end_format_dict + except Exception: + pass # Not valid URL encoding + + # Try base64 decoding + try: + decoded_base64 = base64.b64decode(value).decode() + if self.is_ascii_printable(decoded_base64): + envelopes.append("base64") + envelopes, end_format_dict = self.recurse_envelopes(decoded_base64, envelopes) + return envelopes, end_format_dict + except (binascii.Error, UnicodeDecodeError): + pass # Not valid base64 + + # Try hex decoding + try: + decoded_hex = bytes.fromhex(value).decode("utf-8") + if self.is_ascii_printable(decoded_hex): + envelopes.append("hex") + envelopes, end_format_dict = self.recurse_envelopes(decoded_hex, envelopes) + return envelopes, end_format_dict + except (ValueError, UnicodeDecodeError): + pass # Not valid hex + + # Try JSON parsing + try: + decoded_json = json.loads(value) + if isinstance(decoded_json, (str, dict, list)): + return envelopes, {"json": decoded_json} + except json.JSONDecodeError: + pass # Not valid JSON + + # Try XML parsing + try: + decoded_xml = ET.fromstring(value) + # Pass 'decoded_xml' to 'xml_to_dict' + xml_dict = self.xml_to_dict(decoded_xml) # Pass decoded XML as the 'elem' argument + return envelopes, {"xml": xml_dict} # Store as JSON-like dict, not XML + except ET.ParseError: + pass # Not valid XML + + return envelopes, end_format + + def isolate_parameter(self): + """ + Isolate the specified subparameter from the data structure (JSON/XML). + The subparameter is accessed using dot notation for nested keys. + """ + if self.end_format_data and self.end_format_subparameter: + # Split the dot notation string into keys + keys = self.end_format_subparameter.split(".") + + # Traverse the nested structure using the keys + subparameter_value = self.end_format_data + for key in keys: + if isinstance(subparameter_value, dict): + subparameter_value = subparameter_value.get(key) + else: + # If the structure is broken (not a dict), return None + return None + + return subparameter_value + + return None + + def update_json_parameter(self, new_value): + """ + Update the specified subparameter in the JSON structure and rebuild it. + """ + # Work with a copy to avoid modifying the original `end_format_data` + end_format_data_copy = deepcopy(self.end_format_data) + + if end_format_data_copy: + end_format_data_copy[self.end_format_subparameter] = new_value + return json.dumps(end_format_data_copy) + return new_value + + def update_xml_parameter(self, new_value): + """ + Convert the JSON-like structure back into an XML string after updating the specific parameter. + """ + if self.end_format_data and self.end_format_subparameter: + # Split the dot notation into keys + keys = self.end_format_subparameter.split(".") + + # Traverse the nested dictionary using the keys to find the target subparameter + current_data = self.end_format_data + for key in keys[:-1]: # Traverse up to the second-to-last key + current_data = current_data.get(key, {}) + + # Update the target subparameter with the new value + if isinstance(current_data, dict): + current_data[keys[-1]] = new_value + + # Convert the JSON-like dict back to an XML string + return self.dict_to_xml(self.end_format_data) + + return new_value + + def to_dict(self): + return { + "envelopes": self.envelopes, + "end_format_type": self.end_format_type, + "end_format_data": self.end_format_data, + "end_format_subparameter": self.end_format_subparameter, + } + + def __getstate__(self): + return self.to_dict() + + def __str__(self): + return f"ParameterEnvelopes(envelopes={self.envelopes}, end_format_type={self.end_format_type}, end_format_data={self.end_format_data}, end_format_subparameter={self.end_format_subparameter})" + + __repr__ = __str__ + + @classmethod + def from_dict(cls, data): + instance = cls() + instance.envelopes = data.get("envelopes", []) + instance.end_format_type = data.get("end_format_type") + instance.end_format_data = data.get("end_format_data") + instance.end_format_subparameter = data.get("end_format_subparameter") + return instance + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if "original_value" in self.data.keys(): + parameterEnvelope_instance = self.ParameterEnvelopes() + parameterEnvelope_instance.initialize_value(self.data["original_value"]) + setattr(self, "envelopes", parameterEnvelope_instance) + + envelopes = getattr(self, "envelopes", None) + if ( + envelopes is not None + and getattr(envelopes, "end_format_type", None) is not None + and getattr(envelopes, "end_format_data", None) + ): + end_format_data = envelopes.end_format_data + + def extract_keys_with_values(data, parent_key=""): + """ + Recursively extract all keys from nested dictionaries that have values (non-empty). + Construct a path-like structure with dot notation (e.g., 'find.search'). + """ + keys = [] + if isinstance(data, dict): + for key, value in data.items(): + # Construct the full key path using dot notation + full_key = f"{parent_key}.{key}" if parent_key else key + + # Only add keys that have non-empty values + if value: + if isinstance(value, dict): + # Recursively check nested dictionaries + keys.extend(extract_keys_with_values(value, full_key)) + else: + # Add the key if it has a non-empty value + keys.append(full_key) + return keys + + # Extract all keys that have non-empty values + end_format_data_keys = extract_keys_with_values(end_format_data) + # If there are keys, assign the first key to end_format_subparameter + if end_format_data_keys: + + # Assign the first key to end_format_subparameter + setattr(envelopes, "end_format_subparameter", end_format_data_keys[0]) + setattr(envelopes, "end_format_subparameter", end_format_data_keys[0]) + + # Iterate through the remaining keys, starting from the second one + for p in end_format_data_keys[1:]: + log.debug(f"generating copy of event for subparameter {p} of type {envelopes.end_format_type}") + + # Make a copy of the current event data + cloned_event = self.clone() + cloned_envelopes = getattr(cloned_event, "envelopes") + cloned_envelopes.end_format_subparameter = p + asyncio.run_coroutine_threadsafe( + self.module.emit_event(cloned_event), asyncio.get_event_loop() + ) + def _data_id(self): # dedupe by url:name:param_type url = self.data.get("url", "") name = self.data.get("name", "") param_type = self.data.get("type", "") - return f"{url}:{name}:{param_type}" + envelopes = getattr(self, "envelopes", None) + subparameter = getattr(envelopes, "end_format_subparameter", "") if envelopes else "" + + return f"{url}:{name}:{param_type}:{subparameter}" + + def _outgoing_dedup_hash(self, event): + + envelopes = self.data.get("envelopes") + return hash( + ( + str(event.host), + event.data["url"], + event.data.get("name", ""), + event.data.get("type", ""), + event.data.get("envelopes", ""), + ) + ) def _url(self): return self.data["url"] @@ -1651,7 +2050,6 @@ def make_event( tags.add("affiliate") event_class = globals().get(event_type, DefaultEvent) - return event_class( data, event_type=event_type, @@ -1697,7 +2095,6 @@ def event_from_json(j, siem_friendly=False): "tags": j.get("tags", []), "confidence": j.get("confidence", 100), "context": j.get("discovery_context", None), - "dummy": True, } if siem_friendly: data = j["data"][event_type] diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 4e45ce21e..6cb3bf913 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -3,6 +3,7 @@ import copy import json import math +import base64 import random import string import asyncio @@ -11,6 +12,7 @@ import ahocorasick import regex as re import subprocess as sp + from pathlib import Path from contextlib import suppress from unidecode import unidecode # noqa F401 diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 24d53286f..1bbb2926a 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -116,8 +116,12 @@ # For use with excavate paramaters extractor input_tag_regex = re.compile( - r"]+?name=[\"\']?([\.$\w]+)[\"\']?(?:[^>]*?value=[\"\']([=+\/\w]*)[\"\'])?[^>]*>" + r"]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>" ) +input_tag_regex2 = re.compile( + r"]*?value=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?>" +) +input_tag_novalue_regex = re.compile(r"]*\bvalue=)[^>]*?name=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>") # jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") # jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}") # jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}") diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 481b63f58..031498630 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -432,11 +432,13 @@ def extract(self): class GetForm(ParameterExtractorRule): name = "GET Form" discovery_regex = r'/]*\bmethod=["\']?get["\']?[^>]*>.*<\/form>/s nocase' - form_content_regexes = [ - bbot_regexes.input_tag_regex, - bbot_regexes.select_tag_regex, - bbot_regexes.textarea_tag_regex, - ] + form_content_regexes = { + "input_tag_regex": bbot_regexes.input_tag_regex, + "input_tag_regex2": bbot_regexes.input_tag_regex2, + "input_tag_novalue_regex": bbot_regexes.input_tag_novalue_regex, + "select_tag_regex": bbot_regexes.select_tag_regex, + "textarea_tag_regex": bbot_regexes.textarea_tag_regex, + } extraction_regex = bbot_regexes.get_form_regex output_type = "GETPARAM" @@ -448,16 +450,26 @@ def extract(self): form_action = form_action.lstrip(".") form_parameters = {} - for form_content_regex in self.form_content_regexes: + match_found = False + for form_content_regex_name, form_content_regex in self.form_content_regexes.items(): input_tags = form_content_regex.findall(form_content) + if input_tags: + match_found = True - for parameter_name, original_value in input_tags: - form_parameters[parameter_name] = original_value.strip() + if form_content_regex_name == "input_tag_novalue_regex": + form_parameters[input_tags[0]] = None - for parameter_name, original_value in form_parameters.items(): - yield self.output_type, parameter_name, original_value, form_action, _exclude_key( - form_parameters, parameter_name - ) + else: + if form_content_regex_name == "input_tag_regex2": + input_tags = input_tags = [(b, a) for a, b in input_tags] + + for parameter_name, original_value in input_tags: + form_parameters[parameter_name] = original_value.strip() + + for parameter_name, original_value in form_parameters.items(): + yield self.output_type, parameter_name, original_value, form_action, _exclude_key( + form_parameters, parameter_name + ) class GetForm2(GetForm): extraction_regex = bbot_regexes.get_form_regex2 @@ -515,12 +527,12 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte f"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule" ) endpoint = event.data["url"] if not endpoint else endpoint + path = f"{event.parsed_url.path.lstrip('/')}" url = ( endpoint if endpoint.startswith(("http://", "https://")) - else f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{endpoint}" + else f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{path}{endpoint}" ) - if self.excavate.helpers.validate_parameter(parameter_name, parameter_type): if self.excavate.in_bl(parameter_name) == False: @@ -1003,8 +1015,8 @@ async def handle_event(self, event): self.debug(f"Cookie found without '=': {header_value}") continue else: - cookie_name = header_value.split("=")[0] - cookie_value = header_value.split("=")[1].split(";")[0] + cookie_name, _, remainder = header_value.partition("=") + cookie_value = remainder.split(";")[0] if self.in_bl(cookie_name) == False: self.assigned_cookies[cookie_name] = cookie_value diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 1f724eb9a..b1fcd19b2 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -128,6 +128,16 @@ async def run_submodule(self, submodule, event): if len(submodule_instance.results) > 0: for r in submodule_instance.results: event_data = {"host": str(event.host), "url": event.data["url"], "description": r["description"]} + + envelopes = getattr(event, "envelopes", None) + if envelopes and envelopes.envelopes: + envelope_summary = f'[{"->".join(envelopes.envelopes)}]' + if envelopes.end_format_type: + envelope_summary += f" Format: [{envelopes.end_format_type}] with subparameter [{envelopes.end_format_subparameter}])" + + # Append the envelope summary to the description + event_data["description"] += f" Envelopes: {envelope_summary}" + if r["type"] == "VULNERABILITY": event_data["severity"] = r["severity"] await self.emit_event( diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 5d15f1a19..31438592b 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -1,4 +1,5 @@ import copy +import base64 class BaseLightfuzz: @@ -19,9 +20,10 @@ def additional_params_process(self, additional_params, additional_params_populat return new_additional_params async def send_probe(self, probe): + probe = self.probe_value_outgoing(probe) getparams = {self.event.data["name"]: probe} url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl() - self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}") + self.lightfuzz.critical(f"lightfuzz sending probe with URL: {url}") r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10) if r: return r.text @@ -29,7 +31,7 @@ async def send_probe(self, probe): def compare_baseline( self, event_type, probe, cookies, additional_params_populate_empty=False, speculative_mode="GETPARAM" ): - + probe = self.probe_value_outgoing(probe) http_compare = None if event_type == "SPECULATIVE": @@ -55,7 +57,7 @@ def compare_baseline( self.event.data["url"], include_cache_buster=False, headers=headers, cookies=cookies ) elif event_type == "POSTPARAM": - data = {self.event.data["name"]: f"{probe}"} + data = {self.probe_value_outgoing(self.event.data["name"]): f"{probe}"} if self.event.data["additional_params"] is not None: data.update( self.additional_params_process( @@ -67,6 +69,21 @@ def compare_baseline( ) return http_compare + async def baseline_probe(self, cookies): + if self.event.data.get("eventtype") == "POSTPARAM": + method = "POST" + else: + method = "GET" + + return await self.lightfuzz.helpers.request( + method=method, + cookies=cookies, + url=self.event.data.get("url"), + allow_redirects=False, + retries=1, + timeout=10, + ) + async def compare_probe( self, http_compare, @@ -77,6 +94,8 @@ async def compare_probe( additional_params_override={}, speculative_mode="GETPARAM", ): + + probe = self.probe_value_outgoing(probe) additional_params = copy.deepcopy(self.event.data.get("additional_params", {})) if additional_params_override: for k, v in additional_params_override.items(): @@ -109,18 +128,20 @@ async def standard_probe( self, event_type, cookies, - probe_value, + probe, timeout=10, additional_params_populate_empty=False, speculative_mode="GETPARAM", ): + probe = self.probe_value_outgoing(probe) + if event_type == "SPECULATIVE": event_type = speculative_mode method = "GET" if event_type == "GETPARAM": - url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}" + url = f"{self.event.data['url']}?{self.event.data['name']}={probe}" if "additional_params" in self.event.data.keys() and self.event.data["additional_params"] is not None: url = self.lightfuzz.helpers.add_get_params( url, self.event.data["additional_params"], encode=False @@ -128,15 +149,15 @@ async def standard_probe( else: url = self.event.data["url"] if event_type == "COOKIE": - cookies_probe = {self.event.data["name"]: probe_value} + cookies_probe = {self.event.data["name"]: probe} cookies = {**cookies, **cookies_probe} if event_type == "HEADER": - headers = {self.event.data["name"]: probe_value} + headers = {self.event.data["name"]: probe} else: headers = {} if event_type == "POSTPARAM": method = "POST" - data = {self.event.data["name"]: probe_value} + data = {self.event.data["name"]: probe} if self.event.data["additional_params"] is not None: data.update( self.additional_params_process( @@ -166,10 +187,22 @@ def metadata(self): ) return metadata_string - def probe_value(self, populate_empty=True): + def probe_value_incoming(self, populate_empty=True): probe_value = str(self.event.data.get("original_value", "")) if (probe_value is None or len(probe_value) == 0) and populate_empty == True: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) - + self.lightfuzz.debug(f"probe_value_incoming (before modification): {probe_value}") + envelopes_instance = getattr(self.event, "envelopes", None) + self.lightfuzz.hugesuccess(envelopes_instance) + probe_value = envelopes_instance.remove_envelopes(probe_value) + self.lightfuzz.debug(f"probe_value_incoming (after modification): {probe_value}") return probe_value + + def probe_value_outgoing(self, outgoing_probe_value): + self.lightfuzz.debug(f"probe_value_outgoing (before modification): {outgoing_probe_value}") + envelopes_instance = getattr(self.event, "envelopes", None) + self.lightfuzz.hugesuccess(envelopes_instance) + outgoing_probe_value = envelopes_instance.add_envelopes(outgoing_probe_value) + self.lightfuzz.debug(f"probe_value_outgoing (after modification): {outgoing_probe_value}") + return outgoing_probe_value diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index 92385c643..86504ac74 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -7,9 +7,8 @@ class CmdILightfuzz(BaseLightfuzz): async def fuzz(self): - cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value() + probe_value = self.probe_value_incoming() canary = self.lightfuzz.helpers.rand_string(8, numeric_only=True) http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 9141c240d..da7e52a63 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -194,7 +194,7 @@ async def padding_oracle(self, probe_value, cookies): } ) - async def error_string_search(self, text_dict): + async def error_string_search(self, text_dict, baseline_text): matching_techniques = set() matching_strings = set() @@ -206,12 +206,18 @@ async def error_string_search(self, text_dict): matching_techniques.add(label) context = f"Lightfuzz Cryptographic Probe Submodule detected a cryptographic error after manipulating parameter: [{self.event.data['name']}]" if len(matching_strings) > 0: - self.results.append( - { - "type": "FINDING", - "description": f"Possible Cryptographic Error. {self.metadata()} Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]", - "context": context, - } + false_positive_check = self.lightfuzz.helpers.string_scan(self.crypto_error_strings, baseline_text) + false_positive_matches = set(matched_strings) & set(false_positive_check) + if not false_positive_matches: + self.results.append( + { + "type": "FINDING", + "description": f"Possible Cryptographic Error. {self.metadata()} Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]", + "context": context, + } + ) + self.lightfuzz.debug( + f"Aborting cryptographic error reporting - baseline_text already contained detected string(s) ({','.join(false_positive_check)})" ) @staticmethod @@ -229,13 +235,20 @@ def identify_hash_function(hash_bytes): return hash_functions[hash_length] async def fuzz(self): + cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value(populate_empty=False) + probe_value = self.probe_value_incoming(populate_empty=False) if not probe_value: self.lightfuzz.debug( f"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" ) return + + baseline_probe = await self.baseline_probe(cookies) + if not baseline_probe: + self.lightfuzz.critical(f"Couldn't get baseline_probe for url {self.event.data['url']}, aborting") + return + try: truncate_probe_value = self.modify_string(probe_value, action="truncate") mutate_probe_value = self.modify_string(probe_value, action="mutate") @@ -290,7 +303,7 @@ async def fuzz(self): # Cryptographic Error String Test await self.error_string_search( - {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text} + {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text}, baseline_probe.text ) if confirmed_techniques or ( diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index d4834221d..ab63d278b 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -9,7 +9,7 @@ class PathTraversalLightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value(populate_empty=False) + probe_value = self.probe_value_incoming(populate_empty=False) if not probe_value: self.lightfuzz.debug( f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" diff --git a/bbot/modules/lightfuzz_submodules/serial.py b/bbot/modules/lightfuzz_submodules/serial.py index 6081a550d..c9939f6f7 100644 --- a/bbot/modules/lightfuzz_submodules/serial.py +++ b/bbot/modules/lightfuzz_submodules/serial.py @@ -4,7 +4,6 @@ class SerialLightfuzz(BaseLightfuzz): async def fuzz(self): - cookies = self.event.data.get("assigned_cookies", {}) control_payload = "DEADBEEFCAFEBABE1234567890ABCDEF" serialization_payloads = { @@ -27,7 +26,7 @@ async def fuzz(self): "java.io.optionaldataexception", ] - probe_value = self.probe_value(populate_empty=False) + probe_value = self.probe_value_incoming(populate_empty=False) if probe_value: self.lightfuzz.debug( f"The Serialization Submodule only operates when there if no original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 9d5db9518..ac660fc4c 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -29,14 +29,7 @@ def evaluate_delay(self, mean_baseline, measured_delay): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - - # custom probe_value generation - if "original_value" in self.event.data and self.event.data["original_value"] is not None: - probe_value = urllib.parse.quote(str(self.event.data["original_value"]), safe="") - - else: - probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) - + probe_value = self.probe_value_incoming(populate_empty=True) http_compare = self.compare_baseline( self.event.data["type"], probe_value, cookies, additional_params_populate_empty=True ) diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml index 16caecbca..a16c43872 100644 --- a/bbot/presets/web/lightfuzz-intense.yml +++ b/bbot/presets/web/lightfuzz-intense.yml @@ -19,6 +19,6 @@ config: modules: lightfuzz: force_common_headers: False - submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] + enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss] excavate: retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-max.yml b/bbot/presets/web/lightfuzz-max.yml index e0c7a738b..0b8fbb23c 100644 --- a/bbot/presets/web/lightfuzz-max.yml +++ b/bbot/presets/web/lightfuzz-max.yml @@ -19,6 +19,6 @@ config: modules: lightfuzz: force_common_headers: True - submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] + enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss] excavate: retain_querystring: True \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-xss.yml b/bbot/presets/web/lightfuzz-xss.yml index fd13b2c67..a0162bce2 100644 --- a/bbot/presets/web/lightfuzz-xss.yml +++ b/bbot/presets/web/lightfuzz-xss.yml @@ -12,4 +12,4 @@ config: spider_depth: 5 modules: lightfuzz: - submodules_enabled: [xss] \ No newline at end of file + enabled_submodules: [xss] \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml index 85cce074b..bde8e7a56 100644 --- a/bbot/presets/web/lightfuzz.yml +++ b/bbot/presets/web/lightfuzz.yml @@ -13,7 +13,7 @@ config: spider_depth: 4 modules: lightfuzz: - submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss] + enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss] disable_post: True excavate: retain_querystring: True diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 71ab9fd64..551dd3b03 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -636,6 +636,29 @@ def check(self, module_test, events): assert excavate_xml_extraction, "Excavate failed to extract xml parameter" +class TestExcavateParameterExtraction_inputtagnovalue(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + + # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER + modules_overrides = ["httpx", "excavate", "hunt"] + getparam_extract_html = """ + + """ + + async def setup_after_prep(self, module_test): + respond_args = {"response_data": self.getparam_extract_html, "headers": {"Content-Type": "text/html"}} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + excavate_getparam_extraction = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [novalue] (GET Form Submodule)": + excavate_getparam_extraction = True + assert excavate_getparam_extraction, "Excavate failed to extract web parameter" + + class excavateTestRule(ExcavateRule): yara_rules = { "SearchForText": 'rule SearchForText { meta: description = "Contains the text AAAABBBBCCCC" strings: $text = "AAAABBBBCCCC" condition: $text }', diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 4c5472130..eb5607085 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -1,9 +1,13 @@ +import json import re +import base64 from .base import ModuleTestBase, tempwordlist from werkzeug.wrappers import Response from urllib.parse import unquote +import xml.etree.ElementTree as ET + # Path Traversal single dot tolerance class Test_Lightfuzz_path_singledot(ModuleTestBase): @@ -231,6 +235,212 @@ def check(self, module_test, events): assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" +# Base64 Envelope XSS Detection +class Test_Lightfuzz_envelope_base64(Test_Lightfuzz_xss): + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + print("****") + print(qs) + + parameter_block = """ +
+
+ + +
+
+ """ + if "search=" in qs: + value = qs.split("search=")[1] + if "&" in value: + value = value.split("&")[0] + xss_block = f""" +
+

0 search results for '{unquote(base64.b64decode(value))}'

+
+
+ """ + print("XSS BLOCK:") + print(xss_block) + return Response(xss_block, status=200) + return Response(parameter_block, status=200) + + def check(self, module_test, events): + + web_parameter_emitted = False + xss_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if ( + "Possible Reflected XSS. Parameter: [search] Context: [Between Tags (z tag)" + in e.data["description"] + ): + xss_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" + + +# Hex Envelope XSS Detection +class Test_Lightfuzz_envelope_hex(Test_Lightfuzz_envelope_base64): + def request_handler(self, request): + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + + if "search=" in qs: + value = qs.split("search=")[1] + if "&" in value: + value = value.split("&")[0] + + try: + # Decode the hex value + decoded_value = bytes.fromhex(unquote(value)).decode() + print(f"Decoded hex value: {decoded_value}") + + # Parse the decoded value as JSON + json_data = json.loads(decoded_value) + + # Extract the desired parameter from the JSON (e.g., 'search') + if "search" in json_data: + extracted_value = json_data["search"] + else: + extracted_value = "[Parameter not found in JSON]" + + except (json.JSONDecodeError, ValueError) as e: + extracted_value = "[Invalid hex or JSON format]" + + xss_block = f""" +
+

0 search results for '{extracted_value}'

+
+
+ """ + print("XSS BLOCK:") + print(xss_block) + return Response(xss_block, status=200) + + return Response(parameter_block, status=200) + + +# Base64 (JSON) Envelope XSS Detection +class Test_Lightfuzz_envelope_jsonb64(Test_Lightfuzz_envelope_base64): + def request_handler(self, request): + qs = str(request.query_string.decode()) + + print("****") + print(qs) + + parameter_block = """ +
+
+ + +
+
+ """ + + if "search=" in qs: + value = qs.split("search=")[1] + if "&" in value: + value = value.split("&")[0] + + try: + # Base64 decode the value + decoded_value = base64.b64decode(unquote(value)).decode() + print(f"Decoded base64 value: {decoded_value}") + + # Parse the decoded value as JSON + json_data = json.loads(decoded_value) + + # Extract the desired parameter from the JSON (e.g., 'search') + if "search" in json_data: + extracted_value = json_data["search"] + else: + extracted_value = "[Parameter not found in JSON]" + + except (json.JSONDecodeError, base64.binascii.Error) as e: + extracted_value = "[Invalid base64 or JSON format]" + + xss_block = f""" +
+

0 search results for '{extracted_value}'

+
+
+ """ + print("XSS BLOCK:") + print(xss_block) + return Response(xss_block, status=200) + + return Response(parameter_block, status=200) + + +# Base64 (XML) Envelope XSS Detection +class Test_Lightfuzz_envelope_xmlb64(Test_Lightfuzz_envelope_base64): + def request_handler(self, request): + qs = str(request.query_string.decode()) + + print("****") + print(qs) + + parameter_block = """ +
+
+ + +
+
+ """ + + if "search=" in qs: + value = qs.split("search=")[1] + if "&" in value: + value = value.split("&")[0] + + try: + # Base64 decode the value + decoded_value = base64.b64decode(unquote(value)).decode() + print(f"Decoded base64 value: {decoded_value}") + + # Parse the decoded value as XML + root = ET.fromstring(decoded_value) + + # Extract the desired parameter from the XML (e.g., 'search') + search_element = root.find(".//search") + if search_element is not None: + extracted_value = search_element.text + else: + extracted_value = "[Parameter not found in XML]" + + except (ET.ParseError, base64.binascii.Error) as e: + extracted_value = "[Invalid base64 or XML format]" + + xss_block = f""" +
+

0 search results for '{extracted_value}'

+
+
+ """ + print("XSS BLOCK:") + print(xss_block) + return Response(xss_block, status=200) + + return Response(parameter_block, status=200) + + # In Tag Attribute XSS Detection class Test_Lightfuzz_xss_intag(Test_Lightfuzz_xss): def request_handler(self, request): @@ -1101,3 +1311,110 @@ def check(self, module_test, events): assert excavate_json_extraction, "Excavate failed to extract json parameter" assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" + + +class Test_Lightfuzz_crypto_error(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["httpx", "excavate", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": {"enabled_submodules": ["crypto"]}, + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + crypto_block = f""" +
+

Access Denied!

+
+
+ """ + if "secret=" in qs: + value = qs.split("=")[1] + if value: + return Response(crypto_block, status=200) + + return Response(parameter_block, status=200) + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + cryptoerror_parameter_extracted = False + cryptoerror_finding_emitted = False + + for e in events: + print(e) + print(e.type) + + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [secret] (GET Form Submodule)" in e.data["description"]: + cryptoerror_parameter_extracted = True + if e.type == "FINDING": + if ( + "Possible Cryptographic Error. Parameter: [secret] Parameter Type: [GETPARAM] Original Value: [08a5a2cea9c5a5576e6e5314edcba581d21c7111c9c0c06990327b9127058d67]" + in e.data["description"] + ): + cryptoerror_finding_emitted = True + assert cryptoerror_parameter_extracted, "Parameter not extracted" + assert cryptoerror_finding_emitted, "Crypto Error Message FINDING not emitted" + + +class Test_Lightfuzz_crypto_error_falsepositive(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["httpx", "excavate", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": {"enabled_submodules": ["crypto"]}, + }, + } + + def request_handler(self, request): + fp_block = """ +
+
+ + +
+

Access Denied!

+
+ """ + return Response(fp_block, status=200) + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + cryptoerror_parameter_extracted = False + cryptoerror_finding_emitted = False + + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [secret] (GET Form Submodule)" in e.data["description"]: + cryptoerror_parameter_extracted = True + if e.type == "FINDING": + if "Possible Cryptographic Error" in e.data["description"]: + cryptoerror_finding_emitted = True + assert cryptoerror_parameter_extracted, "Parameter not extracted" + assert ( + not cryptoerror_finding_emitted + ), "Crypto Error Message FINDING was emitted (it is an intentional false positive)" From 5f126f976806a000b1467c774262371e4cae42ce Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:23:23 -0400 Subject: [PATCH 159/238] flake8 --- bbot/core/event/base.py | 7 ------- bbot/core/helpers/misc.py | 1 - bbot/modules/internal/excavate.py | 2 -- bbot/modules/lightfuzz_submodules/base.py | 2 -- bbot/modules/lightfuzz_submodules/sqli.py | 2 -- .../test/test_step_2/module_tests/test_module_lightfuzz.py | 6 +++--- 6 files changed, 3 insertions(+), 17 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index b096aa095..a50fe0278 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -5,7 +5,6 @@ import uuid import json import base64 -import inspect import asyncio import logging import tarfile @@ -1393,10 +1392,6 @@ def _build_tree(element, subdict): return ET.tostring(root_element, encoding="utf-8").decode("utf-8") - @staticmethod - def is_ascii_printable(s): - return all(32 <= ord(char) < 127 for char in s) - preprocess_map = { "base64": preprocess_base64, "hex": preprocess_hex, @@ -1675,8 +1670,6 @@ def _data_id(self): return f"{url}:{name}:{param_type}:{subparameter}" def _outgoing_dedup_hash(self, event): - - envelopes = self.data.get("envelopes") return hash( ( str(event.host), diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 6cb3bf913..6c8733eed 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -3,7 +3,6 @@ import copy import json import math -import base64 import random import string import asyncio diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 031498630..bf8aea908 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -450,11 +450,9 @@ def extract(self): form_action = form_action.lstrip(".") form_parameters = {} - match_found = False for form_content_regex_name, form_content_regex in self.form_content_regexes.items(): input_tags = form_content_regex.findall(form_content) if input_tags: - match_found = True if form_content_regex_name == "input_tag_novalue_regex": form_parameters[input_tags[0]] = None diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 31438592b..b1622270c 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -1,6 +1,4 @@ import copy -import base64 - class BaseLightfuzz: def __init__(self, lightfuzz, event): diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index ac660fc4c..cf10aff9d 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -1,10 +1,8 @@ from .base import BaseLightfuzz from bbot.errors import HttpCompareError -import urllib.parse import statistics - class SQLiLightfuzz(BaseLightfuzz): expected_delay = 5 diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index eb5607085..cdbe9ef4c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -320,7 +320,7 @@ def request_handler(self, request): else: extracted_value = "[Parameter not found in JSON]" - except (json.JSONDecodeError, ValueError) as e: + except (json.JSONDecodeError, ValueError): extracted_value = "[Invalid hex or JSON format]" xss_block = f""" @@ -372,7 +372,7 @@ def request_handler(self, request): else: extracted_value = "[Parameter not found in JSON]" - except (json.JSONDecodeError, base64.binascii.Error) as e: + except (json.JSONDecodeError, base64.binascii.Error): extracted_value = "[Invalid base64 or JSON format]" xss_block = f""" @@ -425,7 +425,7 @@ def request_handler(self, request): else: extracted_value = "[Parameter not found in XML]" - except (ET.ParseError, base64.binascii.Error) as e: + except (ET.ParseError, base64.binascii.Error): extracted_value = "[Invalid base64 or XML format]" xss_block = f""" From b1fabdd311ab8b7a63547d691cdc8d8fd4b372e2 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:27:08 -0400 Subject: [PATCH 160/238] black --- bbot/modules/lightfuzz_submodules/base.py | 1 + bbot/modules/lightfuzz_submodules/sqli.py | 1 + 2 files changed, 2 insertions(+) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index b1622270c..51e93cc50 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -1,5 +1,6 @@ import copy + class BaseLightfuzz: def __init__(self, lightfuzz, event): self.lightfuzz = lightfuzz diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index cf10aff9d..0cf17975e 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -3,6 +3,7 @@ import statistics + class SQLiLightfuzz(BaseLightfuzz): expected_delay = 5 From 2f6062495a33156258e27342ece409da3d05437e Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:29:36 -0400 Subject: [PATCH 161/238] clean up debug --- bbot/modules/lightfuzz_submodules/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 51e93cc50..05d1dbbe0 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -22,7 +22,7 @@ async def send_probe(self, probe): probe = self.probe_value_outgoing(probe) getparams = {self.event.data["name"]: probe} url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl() - self.lightfuzz.critical(f"lightfuzz sending probe with URL: {url}") + self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}") r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10) if r: return r.text @@ -193,7 +193,6 @@ def probe_value_incoming(self, populate_empty=True): probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) self.lightfuzz.debug(f"probe_value_incoming (before modification): {probe_value}") envelopes_instance = getattr(self.event, "envelopes", None) - self.lightfuzz.hugesuccess(envelopes_instance) probe_value = envelopes_instance.remove_envelopes(probe_value) self.lightfuzz.debug(f"probe_value_incoming (after modification): {probe_value}") return probe_value @@ -201,7 +200,6 @@ def probe_value_incoming(self, populate_empty=True): def probe_value_outgoing(self, outgoing_probe_value): self.lightfuzz.debug(f"probe_value_outgoing (before modification): {outgoing_probe_value}") envelopes_instance = getattr(self.event, "envelopes", None) - self.lightfuzz.hugesuccess(envelopes_instance) outgoing_probe_value = envelopes_instance.add_envelopes(outgoing_probe_value) self.lightfuzz.debug(f"probe_value_outgoing (after modification): {outgoing_probe_value}") return outgoing_probe_value From 07eccb5a3ee1a2c52df85198cd400e5e70e40051 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:33:54 -0400 Subject: [PATCH 162/238] remove debug --- .../test_step_2/module_tests/test_module_lightfuzz.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index cdbe9ef4c..e460d2d2b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -262,8 +262,6 @@ def request_handler(self, request):
""" - print("XSS BLOCK:") - print(xss_block) return Response(xss_block, status=200) return Response(parameter_block, status=200) @@ -329,8 +327,6 @@ def request_handler(self, request):
""" - print("XSS BLOCK:") - print(xss_block) return Response(xss_block, status=200) return Response(parameter_block, status=200) @@ -381,8 +377,6 @@ def request_handler(self, request):
""" - print("XSS BLOCK:") - print(xss_block) return Response(xss_block, status=200) return Response(parameter_block, status=200) @@ -434,8 +428,6 @@ def request_handler(self, request):
""" - print("XSS BLOCK:") - print(xss_block) return Response(xss_block, status=200) return Response(parameter_block, status=200) From 4ca9b796ff67cfac963291b4062fd4e6950caafd Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:41:13 -0400 Subject: [PATCH 163/238] update poetry.lock --- poetry.lock | 191 ++++++++++++++++++++++++++-------------------------- 1 file changed, 96 insertions(+), 95 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3094753ce..2b7f97016 100644 --- a/poetry.lock +++ b/poetry.lock @@ -131,33 +131,33 @@ lxml = ["lxml"] [[package]] name = "black" -version = "24.8.0" +version = "24.10.0" description = "The uncompromising code formatter." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "black-24.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09cdeb74d494ec023ded657f7092ba518e8cf78fa8386155e4a03fdcc44679e6"}, - {file = "black-24.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81c6742da39f33b08e791da38410f32e27d632260e599df7245cccee2064afeb"}, - {file = "black-24.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:707a1ca89221bc8a1a64fb5e15ef39cd755633daa672a9db7498d1c19de66a42"}, - {file = "black-24.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:d6417535d99c37cee4091a2f24eb2b6d5ec42b144d50f1f2e436d9fe1916fe1a"}, - {file = "black-24.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fb6e2c0b86bbd43dee042e48059c9ad7830abd5c94b0bc518c0eeec57c3eddc1"}, - {file = "black-24.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:837fd281f1908d0076844bc2b801ad2d369c78c45cf800cad7b61686051041af"}, - {file = "black-24.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62e8730977f0b77998029da7971fa896ceefa2c4c4933fcd593fa599ecbf97a4"}, - {file = "black-24.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:72901b4913cbac8972ad911dc4098d5753704d1f3c56e44ae8dce99eecb0e3af"}, - {file = "black-24.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7c046c1d1eeb7aea9335da62472481d3bbf3fd986e093cffd35f4385c94ae368"}, - {file = "black-24.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:649f6d84ccbae73ab767e206772cc2d7a393a001070a4c814a546afd0d423aed"}, - {file = "black-24.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b59b250fdba5f9a9cd9d0ece6e6d993d91ce877d121d161e4698af3eb9c1018"}, - {file = "black-24.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:6e55d30d44bed36593c3163b9bc63bf58b3b30e4611e4d88a0c3c239930ed5b2"}, - {file = "black-24.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:505289f17ceda596658ae81b61ebbe2d9b25aa78067035184ed0a9d855d18afd"}, - {file = "black-24.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b19c9ad992c7883ad84c9b22aaa73562a16b819c1d8db7a1a1a49fb7ec13c7d2"}, - {file = "black-24.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f13f7f386f86f8121d76599114bb8c17b69d962137fc70efe56137727c7047e"}, - {file = "black-24.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:f490dbd59680d809ca31efdae20e634f3fae27fba3ce0ba3208333b713bc3920"}, - {file = "black-24.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eab4dd44ce80dea27dc69db40dab62d4ca96112f87996bca68cd75639aeb2e4c"}, - {file = "black-24.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3c4285573d4897a7610054af5a890bde7c65cb466040c5f0c8b732812d7f0e5e"}, - {file = "black-24.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e84e33b37be070ba135176c123ae52a51f82306def9f7d063ee302ecab2cf47"}, - {file = "black-24.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:73bbf84ed136e45d451a260c6b73ed674652f90a2b3211d6a35e78054563a9bb"}, - {file = "black-24.8.0-py3-none-any.whl", hash = "sha256:972085c618ee94f402da1af548a4f218c754ea7e5dc70acb168bfaca4c2542ed"}, - {file = "black-24.8.0.tar.gz", hash = "sha256:2500945420b6784c38b9ee885af039f5e7471ef284ab03fa35ecdde4688cd83f"}, + {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, + {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, + {file = "black-24.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:649fff99a20bd06c6f727d2a27f401331dc0cc861fb69cde910fe95b01b5928f"}, + {file = "black-24.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:fe4d6476887de70546212c99ac9bd803d90b42fc4767f058a0baa895013fbb3e"}, + {file = "black-24.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a2221696a8224e335c28816a9d331a6c2ae15a2ee34ec857dcf3e45dbfa99ad"}, + {file = "black-24.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9da3333530dbcecc1be13e69c250ed8dfa67f43c4005fb537bb426e19200d50"}, + {file = "black-24.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4007b1393d902b48b36958a216c20c4482f601569d19ed1df294a496eb366392"}, + {file = "black-24.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:394d4ddc64782e51153eadcaaca95144ac4c35e27ef9b0a42e121ae7e57a9175"}, + {file = "black-24.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e39e0fae001df40f95bd8cc36b9165c5e2ea88900167bddf258bacef9bbdc3"}, + {file = "black-24.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d37d422772111794b26757c5b55a3eade028aa3fde43121ab7b673d050949d65"}, + {file = "black-24.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14b3502784f09ce2443830e3133dacf2c0110d45191ed470ecb04d0f5f6fcb0f"}, + {file = "black-24.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:30d2c30dc5139211dda799758559d1b049f7f14c580c409d6ad925b74a4208a8"}, + {file = "black-24.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cbacacb19e922a1d75ef2b6ccaefcd6e93a2c05ede32f06a21386a04cedb981"}, + {file = "black-24.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1f93102e0c5bb3907451063e08b9876dbeac810e7da5a8bfb7aeb5a9ef89066b"}, + {file = "black-24.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddacb691cdcdf77b96f549cf9591701d8db36b2f19519373d60d31746068dbf2"}, + {file = "black-24.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:680359d932801c76d2e9c9068d05c6b107f2584b2a5b88831c83962eb9984c1b"}, + {file = "black-24.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:17374989640fbca88b6a448129cd1745c5eb8d9547b464f281b251dd00155ccd"}, + {file = "black-24.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:63f626344343083322233f175aaf372d326de8436f5928c042639a4afbbf1d3f"}, + {file = "black-24.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfa1d0cb6200857f1923b602f978386a3a2758a65b52e0950299ea014be6800"}, + {file = "black-24.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:2cd9c95431d94adc56600710f8813ee27eea544dd118d45896bb734e9d7a0dc7"}, + {file = "black-24.10.0-py3-none-any.whl", hash = "sha256:3bb2b7a1f7b685f85b11fed1ef10f8a9148bceb49853e47a294a3dd963c1dd7d"}, + {file = "black-24.10.0.tar.gz", hash = "sha256:846ea64c97afe3bc677b761787993be4991810ecc7a4a937816dd6bddedc4875"}, ] [package.dependencies] @@ -171,7 +171,7 @@ typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] +d = ["aiohttp (>=3.10)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] @@ -595,21 +595,21 @@ files = [ [[package]] name = "dnspython" -version = "2.6.1" +version = "2.7.0" description = "DNS toolkit" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "dnspython-2.6.1-py3-none-any.whl", hash = "sha256:5ef3b9680161f6fa89daf8ad451b5f1a33b18ae8a1c6778cdf4b43f08c0a6e50"}, - {file = "dnspython-2.6.1.tar.gz", hash = "sha256:e8f0f9c23a7b7cb99ded64e6c3a6f3e701d78f50c55e002b839dea7225cff7cc"}, + {file = "dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86"}, + {file = "dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1"}, ] [package.extras] -dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "sphinx (>=7.2.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] -dnssec = ["cryptography (>=41)"] +dev = ["black (>=23.1.0)", "coverage (>=7.0)", "flake8 (>=7)", "hypercorn (>=0.16.0)", "mypy (>=1.8)", "pylint (>=3)", "pytest (>=7.4)", "pytest-cov (>=4.1.0)", "quart-trio (>=0.11.0)", "sphinx (>=7.2.0)", "sphinx-rtd-theme (>=2.0.0)", "twine (>=4.0.0)", "wheel (>=0.42.0)"] +dnssec = ["cryptography (>=43)"] doh = ["h2 (>=4.1.0)", "httpcore (>=1.0.0)", "httpx (>=0.26.0)"] -doq = ["aioquic (>=0.9.25)"] -idna = ["idna (>=3.6)"] +doq = ["aioquic (>=1.0.0)"] +idna = ["idna (>=3.7)"] trio = ["trio (>=0.23)"] wmi = ["wmi (>=1.5.1)"] @@ -1076,71 +1076,72 @@ testing = ["coverage", "pyyaml"] [[package]] name = "markupsafe" -version = "2.1.5" +version = "3.0.1" description = "Safely add untrusted strings to HTML/XML markup." optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:db842712984e91707437461930e6011e60b39136c7331e971952bb30465bc1a1"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ffb4a8e7d46ed96ae48805746755fadd0909fea2306f93d5d8233ba23dda12a"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67c519635a4f64e495c50e3107d9b4075aec33634272b5db1cde839e07367589"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48488d999ed50ba8d38c581d67e496f955821dc183883550a6fbc7f1aefdc170"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f31ae06f1328595d762c9a2bf29dafd8621c7d3adc130cbb46278079758779ca"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:80fcbf3add8790caddfab6764bde258b5d09aefbe9169c183f88a7410f0f6dea"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3341c043c37d78cc5ae6e3e305e988532b072329639007fd408a476642a89fd6"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cb53e2a99df28eee3b5f4fea166020d3ef9116fdc5764bc5117486e6d1211b25"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-win32.whl", hash = "sha256:db15ce28e1e127a0013dfb8ac243a8e392db8c61eae113337536edb28bdc1f97"}, + {file = "MarkupSafe-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:4ffaaac913c3f7345579db4f33b0020db693f302ca5137f106060316761beea9"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:26627785a54a947f6d7336ce5963569b5d75614619e75193bdb4e06e21d447ad"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b954093679d5750495725ea6f88409946d69cfb25ea7b4c846eef5044194f583"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:973a371a55ce9ed333a3a0f8e0bcfae9e0d637711534bcb11e130af2ab9334e7"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:244dbe463d5fb6d7ce161301a03a6fe744dac9072328ba9fc82289238582697b"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d98e66a24497637dd31ccab090b34392dddb1f2f811c4b4cd80c230205c074a3"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad91738f14eb8da0ff82f2acd0098b6257621410dcbd4df20aaa5b4233d75a50"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7044312a928a66a4c2a22644147bc61a199c1709712069a344a3fb5cfcf16915"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a4792d3b3a6dfafefdf8e937f14906a51bd27025a36f4b188728a73382231d91"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win32.whl", hash = "sha256:fa7d686ed9883f3d664d39d5a8e74d3c5f63e603c2e3ff0abcba23eac6542635"}, + {file = "MarkupSafe-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:9ba25a71ebf05b9bb0e2ae99f8bc08a07ee8e98c612175087112656ca0f5c8bf"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8ae369e84466aa70f3154ee23c1451fda10a8ee1b63923ce76667e3077f2b0c4"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40f1e10d51c92859765522cbd79c5c8989f40f0419614bcdc5015e7b6bf97fc5"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a4cb365cb49b750bdb60b846b0c0bc49ed62e59a76635095a179d440540c346"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee3941769bd2522fe39222206f6dd97ae83c442a94c90f2b7a25d847d40f4729"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62fada2c942702ef8952754abfc1a9f7658a4d5460fabe95ac7ec2cbe0d02abc"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c2d64fdba74ad16138300815cfdc6ab2f4647e23ced81f59e940d7d4a1469d9"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:fb532dd9900381d2e8f48172ddc5a59db4c445a11b9fab40b3b786da40d3b56b"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0f84af7e813784feb4d5e4ff7db633aba6c8ca64a833f61d8e4eade234ef0c38"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-win32.whl", hash = "sha256:cbf445eb5628981a80f54087f9acdbf84f9b7d862756110d172993b9a5ae81aa"}, + {file = "MarkupSafe-3.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:a10860e00ded1dd0a65b83e717af28845bb7bd16d8ace40fe5531491de76b79f"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e81c52638315ff4ac1b533d427f50bc0afc746deb949210bc85f05d4f15fd772"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:312387403cd40699ab91d50735ea7a507b788091c416dd007eac54434aee51da"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ae99f31f47d849758a687102afdd05bd3d3ff7dbab0a8f1587981b58a76152a"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c97ff7fedf56d86bae92fa0a646ce1a0ec7509a7578e1ed238731ba13aabcd1c"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7420ceda262dbb4b8d839a4ec63d61c261e4e77677ed7c66c99f4e7cb5030dd"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:45d42d132cff577c92bfba536aefcfea7e26efb975bd455db4e6602f5c9f45e7"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4c8817557d0de9349109acb38b9dd570b03cc5014e8aabf1cbddc6e81005becd"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a54c43d3ec4cf2a39f4387ad044221c66a376e58c0d0e971d47c475ba79c6b5"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-win32.whl", hash = "sha256:c91b394f7601438ff79a4b93d16be92f216adb57d813a78be4446fe0f6bc2d8c"}, + {file = "MarkupSafe-3.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:fe32482b37b4b00c7a52a07211b479653b7fe4f22b2e481b9a9b099d8a430f2f"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:17b2aea42a7280db02ac644db1d634ad47dcc96faf38ab304fe26ba2680d359a"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:852dc840f6d7c985603e60b5deaae1d89c56cb038b577f6b5b8c808c97580f1d"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0778de17cff1acaeccc3ff30cd99a3fd5c50fc58ad3d6c0e0c4c58092b859396"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:800100d45176652ded796134277ecb13640c1a537cad3b8b53da45aa96330453"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d06b24c686a34c86c8c1fba923181eae6b10565e4d80bdd7bc1c8e2f11247aa4"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:33d1c36b90e570ba7785dacd1faaf091203d9942bc036118fab8110a401eb1a8"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:beeebf760a9c1f4c07ef6a53465e8cfa776ea6a2021eda0d0417ec41043fe984"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:bbde71a705f8e9e4c3e9e33db69341d040c827c7afa6789b14c6e16776074f5a"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-win32.whl", hash = "sha256:82b5dba6eb1bcc29cc305a18a3c5365d2af06ee71b123216416f7e20d2a84e5b"}, + {file = "MarkupSafe-3.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:730d86af59e0e43ce277bb83970530dd223bf7f2a838e086b50affa6ec5f9295"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4935dd7883f1d50e2ffecca0aa33dc1946a94c8f3fdafb8df5c330e48f71b132"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e9393357f19954248b00bed7c56f29a25c930593a77630c719653d51e7669c2a"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40621d60d0e58aa573b68ac5e2d6b20d44392878e0bfc159012a5787c4e35bc8"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f94190df587738280d544971500b9cafc9b950d32efcb1fba9ac10d84e6aa4e6"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6a387d61fe41cdf7ea95b38e9af11cfb1a63499af2759444b99185c4ab33f5b"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8ad4ad1429cd4f315f32ef263c1342166695fad76c100c5d979c45d5570ed58b"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e24bfe89c6ac4c31792793ad9f861b8f6dc4546ac6dc8f1c9083c7c4f2b335cd"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2a4b34a8d14649315c4bc26bbfa352663eb51d146e35eef231dd739d54a5430a"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-win32.whl", hash = "sha256:242d6860f1fd9191aef5fae22b51c5c19767f93fb9ead4d21924e0bcb17619d8"}, + {file = "MarkupSafe-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:93e8248d650e7e9d49e8251f883eed60ecbc0e8ffd6349e18550925e31bd029b"}, + {file = "markupsafe-3.0.1.tar.gz", hash = "sha256:3e683ee4f5d0fa2dde4db77ed8dd8a876686e3fc417655c2ece9a90576905344"}, ] [[package]] From 68f9b9df7ecf04257675adee8fdee27309b2fb6c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 19:43:36 -0400 Subject: [PATCH 164/238] poetry lock update --- poetry.lock | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index e08c93b19..4299cf69c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1622,13 +1622,13 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"] [[package]] name = "pre-commit" -version = "4.0.0" +version = "4.0.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-4.0.0-py2.py3-none-any.whl", hash = "sha256:0ca2341cf94ac1865350970951e54b1a50521e57b7b500403307aed4315a1234"}, - {file = "pre_commit-4.0.0.tar.gz", hash = "sha256:5d9807162cc5537940f94f266cbe2d716a75cfad0d78a317a92cac16287cfed6"}, + {file = "pre_commit-4.0.1-py2.py3-none-any.whl", hash = "sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878"}, + {file = "pre_commit-4.0.1.tar.gz", hash = "sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2"}, ] [package.dependencies] @@ -3110,5 +3110,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "1bd4ecfe101d19e182401daf146486d6bcba963b802de78d8f8955454eb992b5" - +content-hash = "93e1608d4f875ac4dcfefbe684df607ea01ca09868e093bfeeb7d2c386375b49" From 9da63bb88c96aace417f77594ea06a6b0285acf9 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 21:02:07 -0400 Subject: [PATCH 165/238] replacing dummy --- bbot/core/event/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index a50fe0278..82528fb88 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -2057,7 +2057,6 @@ def make_event( _internal=internal, ) - def event_from_json(j, siem_friendly=False): """ Creates an event object from a JSON dictionary. @@ -2088,6 +2087,7 @@ def event_from_json(j, siem_friendly=False): "tags": j.get("tags", []), "confidence": j.get("confidence", 100), "context": j.get("discovery_context", None), + "dummy": True, } if siem_friendly: data = j["data"][event_type] From 0d395f923119351cf9cd8ebe14c3f3e585d4592f Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 21:03:59 -0400 Subject: [PATCH 166/238] black --- bbot/core/event/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 82528fb88..40cae7202 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -2057,6 +2057,7 @@ def make_event( _internal=internal, ) + def event_from_json(j, siem_friendly=False): """ Creates an event object from a JSON dictionary. From d66badd14cad63bddd2f78ce93988be0952abd05 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 23:38:32 -0400 Subject: [PATCH 167/238] tweak debug msg --- bbot/core/event/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 40cae7202..ad4a3006f 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1466,7 +1466,7 @@ def recurse_envelopes(self, value, envelopes=None, end_format=None): if envelopes is None: envelopes = [] log.debug( - f"Starting recurse with value: {value}, current envelopes: {', '.join(envelopes)}, current end format: {end_format}" + f"Starting envelope recurse with value: [{value}], current envelopes: [{', '.join(envelopes)}], current end format: {end_format}" ) if value is None or value == "" or isinstance(value, int): @@ -2102,7 +2102,6 @@ def event_from_json(j, siem_friendly=False): resolved_hosts = j.get("resolved_hosts", []) event._resolved_hosts = set(resolved_hosts) - event.timestamp = datetime.datetime.fromisoformat(j["timestamp"]) event.scope_distance = j["scope_distance"] parent_id = j.get("parent", None) From 8ec46cbb93e39454dbc393ec8f02812c1be17421 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 23:39:07 -0400 Subject: [PATCH 168/238] fixing path reconstruction bug --- bbot/modules/internal/excavate.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index bf8aea908..ded56bd77 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -329,6 +329,8 @@ class excavateTestRule(ExcavateRule): "ASP.NET_SessionId", "JSESSIONID", "PHPSESSID", + "AWSALB", + "AWSALBCORS", ] ) @@ -524,13 +526,19 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte self.excavate.debug( f"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule" ) - endpoint = event.data["url"] if not endpoint else endpoint - path = f"{event.parsed_url.path.lstrip('/')}" - url = ( - endpoint - if endpoint.startswith(("http://", "https://")) - else f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{path}{endpoint}" - ) + # If we have a full URL, leave it as-is + if not endpoint.startswith(("http://", "https://")): + + # The endpoint is usually a form action - we should use it if we have it. If not, defautl to URL. + path = event.parsed_url.path if not endpoint else endpoint + # Normalize path by remove leading slash + path = path.lstrip("/") + + # Ensure the base URL has a single slash between path and endpoint + url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{path}" + else: + url = endpoint + if self.excavate.helpers.validate_parameter(parameter_name, parameter_type): if self.excavate.in_bl(parameter_name) == False: From 45c6c1b96e4dd28c6de62a808f1519428db82e8d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 8 Oct 2024 23:42:41 -0400 Subject: [PATCH 169/238] adding additional test --- bbot/modules/lightfuzz_submodules/crypto.py | 2 +- .../module_tests/test_module_excavate.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index da7e52a63..526877339 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -246,7 +246,7 @@ async def fuzz(self): baseline_probe = await self.baseline_probe(cookies) if not baseline_probe: - self.lightfuzz.critical(f"Couldn't get baseline_probe for url {self.event.data['url']}, aborting") + self.lightfuzz.warning(f"Couldn't get baseline_probe for url {self.event.data['url']}, aborting") return try: diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 551dd3b03..814c63493 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -582,6 +582,29 @@ def check(self, module_test, events): assert excavate_getparam_extraction, "Excavate failed to extract web parameter" +class TestExcavateParameterExtraction_getparam_novalue(TestExcavateParameterExtraction_getparam): + getparam_extract_html = """ +
+
+ + + +
+
+ """ + + def check(self, module_test, events): + excavate_getparam_extraction = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [searchTerm] (GET Form Submodule)" in e.data["description"]: + excavate_getparam_extraction = True + assert excavate_getparam_extraction, "Excavate failed to extract web parameter" + + class TestExcavateParameterExtraction_json(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] modules_overrides = ["httpx", "excavate", "paramminer_getparams"] From 069fdc64447ca1c8326d29cb5ec6960b1d633db3 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 9 Oct 2024 07:49:45 -0400 Subject: [PATCH 170/238] 3.9 compatibility hack --- bbot/core/event/base.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index ad4a3006f..ea7ca2aad 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1436,13 +1436,15 @@ def remove_envelopes(self, value): for env in self.envelopes: func = self.preprocess_map.get(env) if func: + # python3.9 compatibility hack + if isinstance(func, staticmethod): + func = func.__get__(None, self.__class__) # Unwrap staticmethod value = func(value) # Dynamically select the appropriate isolate function based on the final format isolate_func = self.format_isolate_map.get(self.end_format_type) if isolate_func: return isolate_func(self) - return value def add_envelopes(self, value): @@ -1453,12 +1455,18 @@ def add_envelopes(self, value): # Dynamically select the appropriate update function based on the final format update_func = self.format_update_map.get(self.end_format_type) if update_func: + # python3.9 compatibility hack + if isinstance(update_func, staticmethod): + update_func = update_func.__get__(None, self.__class__) value = update_func(self, value) # Apply the envelopes in reverse order for env in self.envelopes[::-1]: func = self.postprocess_map.get(env) if func: + # python3.9 compatibility hack + if isinstance(func, staticmethod): + func = func.__get__(None, self.__class__) value = func(value) return value From 9cd281a3cb6698acfc735841b538e159400586a9 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 9 Oct 2024 13:41:41 -0400 Subject: [PATCH 171/238] removing unnecessary threading lock handling --- bbot/core/event/base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index ea7ca2aad..e8429869f 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,7 +1,6 @@ from collections import defaultdict import io import re -import threading import uuid import json import base64 @@ -630,14 +629,10 @@ def clone(self): # Handle attributes that need deep copying manually setattr(cloned_event, "envelopes", deepcopy(self.envelopes)) - # cloned_event.scan = deepcopy(self.scan) # Re-assign a new UUID cloned_event.uuid = uuid.uuid4() - # Re-create the unpickleable lock object - cloned_event.lock = threading.RLock() - return cloned_event def _host(self): From 03c06ca2b6203db11f3d23e3867009cb570a16fa Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 10 Oct 2024 16:28:39 -0400 Subject: [PATCH 172/238] various bug fixes with parameter extraction, and in crypto submodule --- bbot/core/helpers/regexes.py | 8 +- bbot/modules/internal/excavate.py | 36 +++-- bbot/modules/lightfuzz_submodules/base.py | 2 +- bbot/modules/lightfuzz_submodules/crypto.py | 138 +++++++++--------- .../module_tests/test_module_excavate.py | 32 ++++ 5 files changed, 132 insertions(+), 84 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 1bbb2926a..fca6f5a44 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -116,10 +116,10 @@ # For use with excavate paramaters extractor input_tag_regex = re.compile( - r"]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>" + r"]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([%\._=+\/\w]*)[\"\']?[^>]*?>" ) input_tag_regex2 = re.compile( - r"]*?value=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?>" + r"]*?value=[\"\']?([%\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?>" ) input_tag_novalue_regex = re.compile(r"]*\bvalue=)[^>]*?name=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>") # jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") @@ -143,6 +143,10 @@ r"]*\baction=[\"']?([^\s\"'<>]+)[\"']?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>", re.DOTALL, ) +post_form_regex_noaction = re.compile( + r"]*(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)?[^>]*\bmethod=[\"']?[pP][oO][sS][tT][\"']?[^>]*>([\s\S]*?)<\/form>", + re.DOTALL, +) generic_form_regex = re.compile( r"]*\bmethod=)[^>]+(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)[^>]*>([\s\S]*?)<\/form>", re.DOTALL ) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index ded56bd77..004c2c0c5 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -448,7 +448,10 @@ def extract(self): forms = self.extraction_regex.findall(str(self.result)) for form_action, form_content in forms: - if form_action.startswith("./"): + if not form_action: + form_action = None + + elif form_action.startswith("./"): form_action = form_action.lstrip(".") form_parameters = {} @@ -461,7 +464,7 @@ def extract(self): else: if form_content_regex_name == "input_tag_regex2": - input_tags = input_tags = [(b, a) for a, b in input_tags] + input_tags = [(b, a) for a, b in input_tags] for parameter_name, original_value in input_tags: form_parameters[parameter_name] = original_value.strip() @@ -483,6 +486,10 @@ class PostForm(GetForm): class PostForm2(PostForm): extraction_regex = bbot_regexes.post_form_regex2 + class PostForm_NoAction(PostForm): + name = "POST Form (no action)" + extraction_regex = bbot_regexes.post_form_regex_noaction + # underscore ensure generic forms runs last, so it doesn't cause dedupe to stop full form detection class _GenericForm(GetForm): name = "Generic Form" @@ -527,17 +534,18 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte f"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule" ) # If we have a full URL, leave it as-is - if not endpoint.startswith(("http://", "https://")): - # The endpoint is usually a form action - we should use it if we have it. If not, defautl to URL. + if endpoint and endpoint.startswith(("http://", "https://")): + url = endpoint + + else: + # The endpoint is usually a form action - we should use it if we have it. If not, default to URL. path = event.parsed_url.path if not endpoint else endpoint # Normalize path by remove leading slash path = path.lstrip("/") # Ensure the base URL has a single slash between path and endpoint url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{path}" - else: - url = endpoint if self.excavate.helpers.validate_parameter(parameter_name, parameter_type): @@ -925,13 +933,15 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon context = f"excavate's Parameter extractor found a speculative WEB_PARAMETER: {parameter_name} by parsing {source_type} data from {str(event.host)}" await self.emit_event(data, "WEB_PARAMETER", event, context=context) return - - for result in self.yara_rules.match(data=f"{data}\n{decoded_data}"): - rule_name = result.rule - if rule_name in self.yara_preprocess_dict: - await self.yara_preprocess_dict[rule_name](result, event, discovery_context) - else: - self.hugewarning(f"YARA Rule {rule_name} not found in pre-compiled rules") + for data_instance in [data, decoded_data]: + for result in self.yara_rules.match(data=f"{data_instance}"): + rule_name = result.rule + if rule_name == "parameter_extraction" and data_instance == decoded_data: + continue + if rule_name in self.yara_preprocess_dict: + await self.yara_preprocess_dict[rule_name](result, event, discovery_context) + else: + self.hugewarning(f"YARA Rule {rule_name} not found in pre-compiled rules") async def handle_event(self, event): diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 05d1dbbe0..b0654a075 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -56,7 +56,7 @@ def compare_baseline( self.event.data["url"], include_cache_buster=False, headers=headers, cookies=cookies ) elif event_type == "POSTPARAM": - data = {self.probe_value_outgoing(self.event.data["name"]): f"{probe}"} + data = {self.event.data["name"]: f"{probe}"} if self.event.data["additional_params"] is not None: data.update( self.additional_params_process( diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 526877339..5635774bf 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -47,28 +47,28 @@ def is_base64(s): ] @staticmethod - def format_agnostic_decode(input_string): + def format_agnostic_decode(input_string, urldecode=False): encoding = "unknown" - decoded_input = unquote(input_string) - if CryptoLightfuzz.is_hex(decoded_input): - data = bytes.fromhex(decoded_input) + if urldecode: + input_string = unquote(input_string) + if CryptoLightfuzz.is_hex(input_string): + data = bytes.fromhex(input_string) encoding = "hex" - elif CryptoLightfuzz.is_base64(decoded_input): - data = base64.b64decode(decoded_input) + elif CryptoLightfuzz.is_base64(input_string): + data = base64.b64decode(input_string) encoding = "base64" else: data = str return data, encoding @staticmethod - def format_agnostic_encode(data, encoding, urlencode=True): + def format_agnostic_encode(data, encoding, urlencode=False): if encoding == "hex": encoded_data = data.hex() elif encoding == "base64": encoded_data = base64.b64encode(data).decode("utf-8") # base64 encoding returns bytes, decode to string else: raise ValueError("Unsupported encoding type specified") - if urlencode: return quote(encoded_data) return encoded_data @@ -126,73 +126,76 @@ def possible_block_sizes(ciphertext_length): potential_block_sizes = [8, 16] possible_sizes = [] for block_size in potential_block_sizes: - num_blocks = ciphertext_length / block_size - if num_blocks.is_integer() and num_blocks >= 2: + num_blocks = ciphertext_length // block_size + if ciphertext_length % block_size == 0 and num_blocks >= 2: possible_sizes.append(block_size) return possible_sizes - async def padding_oracle_execute(self, original_data, encoding, cookies, possible_first_byte=False): - - possible_block_sizes = self.possible_block_sizes(len(original_data)) - - for block_size in possible_block_sizes: - ivblock = b"\x00" * block_size - paddingblock = b"\x00" * block_size - datablock = original_data[-block_size:] - if possible_first_byte: - baseline_byte = b"\xFF" - starting_pos = 0 - else: - baseline_byte = b"\x00" - starting_pos = 1 - - baseline = self.compare_baseline( - self.event.data["type"], ivblock + paddingblock[:-1] + baseline_byte + datablock, cookies + async def padding_oracle_execute(self, original_data, encoding, block_size, cookies, possible_first_byte=True): + ivblock = b"\x00" * block_size + paddingblock = b"\x00" * block_size + datablock = original_data[-block_size:] + if possible_first_byte: + baseline_byte = b"\xFF" + starting_pos = 0 + else: + baseline_byte = b"\x00" + starting_pos = 1 + baseline = self.compare_baseline( + self.event.data["type"], + self.format_agnostic_encode(ivblock + paddingblock[:-1] + baseline_byte + datablock, encoding), + cookies, + ) + differ_count = 0 + for i in range(starting_pos, starting_pos + 254): + + byte = bytes([i]) + oracle_probe = await self.compare_probe( + baseline, + self.event.data["type"], + self.format_agnostic_encode(ivblock + paddingblock[:-1] + byte + datablock, encoding), + cookies, ) - differ_count = 0 - for i in range(starting_pos, starting_pos + 254): - byte = bytes([i]) - oracle_probe = await self.compare_probe( - baseline, - self.event.data["type"], - self.format_agnostic_encode(ivblock + paddingblock[:-1] + byte + datablock, encoding), - cookies, - ) - - if oracle_probe[0] == False and "body" in oracle_probe[1]: - differ_count += 1 - if i == 1: - possible_first_byte = True - continue - elif i == 2 and possible_first_byte == True: - # Thats two results which appear "different". Its entirely possible \x00 was the correct padding. We will break from this loop and redo it with the last byte as the baseline instead of the first - return False, None - if differ_count == 1: - return True, block_size - else: - continue - return False, None + if oracle_probe[0] == False and "body" in oracle_probe[1]: + differ_count += 1 + + if i == 2: + if possible_first_byte == True: + # Thats two results which appear "different". Since this is the first run, it's entirely possible \x00 was the correct padding. + # We will break from this loop and redo it with the last byte as the baseline instead of the first + return None + else: + # Now that we have tried the run twice, we know it can't be because the first byte was the correct padding, and we know it is not vulnerable + return False + if differ_count == 1: + return True + return False async def padding_oracle(self, probe_value, cookies): data, encoding = self.format_agnostic_decode(probe_value) + possible_block_sizes = self.possible_block_sizes(len(data)) - padding_oracle_result, block_size = await self.padding_oracle_execute(data, encoding, cookies) - if padding_oracle_result == None: - self.lightfuzz.debug("ended up in possible_first_byte situation - retrying with different first byte") - padding_oracle_result = await self.padding_oracle_execute( - data, encoding, cookies, possible_first_byte=True - ) + for block_size in possible_block_sizes: - if padding_oracle_result == True: - context = f"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]" - self.results.append( - { - "type": "VULNERABILITY", - "severity": "HIGH", - "description": f"Padding Oracle Vulnerability. Block size: [{str(block_size)}] {self.metadata()}", - "context": context, - } - ) + padding_oracle_result = await self.padding_oracle_execute(data, encoding, block_size, cookies) + if padding_oracle_result == None: + self.lightfuzz.debug( + "still could be in a possible_first_byte situation - retrying with different first byte" + ) + padding_oracle_result = await self.padding_oracle_execute( + data, encoding, block_size, cookies, possible_first_byte=False + ) + + if padding_oracle_result == True: + context = f"Lightfuzz Cryptographic Probe Submodule detected a probable padding oracle vulnerability after manipulating parameter: [{self.event.data['name']}]" + self.results.append( + { + "type": "VULNERABILITY", + "severity": "HIGH", + "description": f"Padding Oracle Vulnerability. Block size: [{str(block_size)}] {self.metadata()}", + "context": context, + } + ) async def error_string_search(self, text_dict, baseline_text): @@ -235,7 +238,6 @@ def identify_hash_function(hash_bytes): return hash_functions[hash_length] async def fuzz(self): - cookies = self.event.data.get("assigned_cookies", {}) probe_value = self.probe_value_incoming(populate_empty=False) if not probe_value: @@ -259,7 +261,7 @@ async def fuzz(self): return # Basic crypanalysis - likely_crypto, possible_block_cipher = self.cryptanalysis(unquote(probe_value)) + likely_crypto, possible_block_cipher = self.cryptanalysis(probe_value) if not likely_crypto: self.lightfuzz.debug("Parameter value does not appear to be cryptographic, aborting tests") diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 814c63493..5099cfe5b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -557,6 +557,38 @@ def check(self, module_test, events): assert found_htmltags_img, "Did not extract parameter(s) from img-tag" +class TestExcavateParameterExtraction_postformnoaction(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + + # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER + modules_overrides = ["httpx", "excavate", "hunt"] + postformnoaction_extract_html = """ + +

Post for without action

+
+ + +

+ +
+ + """ + + async def setup_after_prep(self, module_test): + respond_args = {"response_data": self.postformnoaction_extract_html, "headers": {"Content-Type": "text/html"}} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + + excavate_getparam_extraction = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [state] (POST Form (no action) Submodule)" in e.data["description"]: + excavate_getparam_extraction = True + assert excavate_getparam_extraction, "Excavate failed to extract web parameter" + + class TestExcavateParameterExtraction_getparam(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] From 0650d928490c5177d20096a96648c03e637df16a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 10 Oct 2024 21:07:40 -0400 Subject: [PATCH 173/238] the worst test i've ever had to make --- bbot/core/helpers/regexes.py | 2 +- bbot/modules/internal/excavate.py | 21 ++++- bbot/modules/lightfuzz_submodules/cmdi.py | 2 +- .../module_tests/test_module_dotnetnuke.py | 4 - .../module_tests/test_module_lightfuzz.py | 80 ++++++++++++++++++- 5 files changed, 97 insertions(+), 12 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index fca6f5a44..af8ddb074 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -157,7 +157,7 @@ textarea_tag_regex = re.compile( r']*\bname=["\']?(\w+)["\']?[^>]*>(.*?)', re.IGNORECASE | re.DOTALL ) -tag_attribute_regex = re.compile(r"<[^>]*(?:href|src)\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>") +tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>") valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+" diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 004c2c0c5..6316a7361 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -901,7 +901,6 @@ async def setup(self): async def search(self, data, event, content_type, discovery_context="HTTP response"): if not data: return None - decoded_data = await self.helpers.re.recursive_decode(data) if self.parameter_extraction: @@ -933,11 +932,27 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon context = f"excavate's Parameter extractor found a speculative WEB_PARAMETER: {parameter_name} by parsing {source_type} data from {str(event.host)}" await self.emit_event(data, "WEB_PARAMETER", event, context=context) return - for data_instance in [data, decoded_data]: + + # Initialize the list of data items to process + data_items = [] + + # Check if data and decoded_data are identical + if data == decoded_data: + data_items.append(("data", data)) # Add only one since both are the same + else: + data_items.append(("data", data)) + data_items.append(("decoded_data", decoded_data)) + + for label, data_instance in data_items: + # Your existing processing code for result in self.yara_rules.match(data=f"{data_instance}"): rule_name = result.rule - if rule_name == "parameter_extraction" and data_instance == decoded_data: + + # Skip specific operations for 'parameter_extraction' rule on decoded_data + if label == "decoded_data" and rule_name == "parameter_extraction": continue + + # Check if rule processing function exists if rule_name in self.yara_preprocess_dict: await self.yara_preprocess_dict[rule_name](result, event, discovery_context) else: diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index 86504ac74..ff2604c17 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -33,7 +33,7 @@ async def fuzz(self): if canary in cmdi_probe[3].text and "echo" not in cmdi_probe[3].text: self.lightfuzz.debug(f"canary [{canary}] found in response when sending probe [{p}]") if p == "AAAA": - self.lightfuzz.hugewarning( + self.lightfuzz.warning( f"False Postive Probe appears to have been triggered for {self.event.data['url']}, aborting remaining detection" ) return diff --git a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py index de78ad50b..78a034f6b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py +++ b/bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py @@ -92,8 +92,6 @@ def check(self, module_test, events): dnn_installwizard_privesc_detection = False for e in events: - print(e) - print(e.type) if e.type == "TECHNOLOGY" and "DotNetNuke" in e.data["technology"]: dnn_technology_detection = True @@ -171,8 +169,6 @@ def check(self, module_test, events): for e in events: - print(e) - print(e.type) if e.type == "TECHNOLOGY" and "DotNetNuke" in e.data["technology"]: dnn_technology_detection = True diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index e460d2d2b..e51fc38e7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -4,7 +4,7 @@ from .base import ModuleTestBase, tempwordlist from werkzeug.wrappers import Response -from urllib.parse import unquote +from urllib.parse import unquote, quote import xml.etree.ElementTree as ET @@ -1351,8 +1351,6 @@ def check(self, module_test, events): cryptoerror_finding_emitted = False for e in events: - print(e) - print(e.type) if e.type == "WEB_PARAMETER": if "HTTP Extracted Parameter [secret] (GET Form Submodule)" in e.data["description"]: @@ -1410,3 +1408,79 @@ def check(self, module_test, events): assert ( not cryptoerror_finding_emitted ), "Crypto Error Message FINDING was emitted (it is an intentional false positive)" + + +class Test_PaddingOracleDetection(ModuleTestBase): + + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "excavate", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "enabled_submodules": ["crypto"], + } + }, + } + + def request_handler(self, request): + encrypted_value = quote( + "dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==" + ) + default_html_response = f""" + + +
+ + +
+ + + """ + + if "/decrypt" in request.url and request.method == "POST": + if request.form and request.form["encrypted_data"]: + encrypted_data = request.form["encrypted_data"] + if "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm" in encrypted_data: + response_content = "Padding error detected" + elif "4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg" in encrypted_data: + response_content = "DIFFERENT CRYPTOGRAPHIC ERROR" + elif "AAAAAAA" in encrypted_data: + response_content = "YET DIFFERENT CRYPTOGRAPHIC ERROR" + else: + response_content = "Decryption failed" + + return Response(response_content, status=200) + else: + return Response(default_html_response, status=200) + + async def setup_after_prep(self, module_test): + module_test.set_expect_requests_handler(expect_args=re.compile(".*"), request_handler=self.request_handler) + + def check(self, module_test, events): + + web_parameter_extracted = False + cryptographic_parameter_finding = False + padding_oracle_detected = False + for e in events: + + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [encrypted_data] (POST Form" in e.data["description"]: + web_parameter_extracted = True + if e.type == "FINDING": + if ( + e.data["description"] + == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [url-encoded]" + ): + cryptographic_parameter_finding = True + + if e.type == "VULNERABILITY": + if ( + e.data["description"] + == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [url-encoded]" + ): + padding_oracle_detected = True + + assert web_parameter_extracted, "Web parameter was not extracted" + assert cryptographic_parameter_finding, "Cryptographic parameter not detected" + assert padding_oracle_detected, "Padding oracle vulnerability was not detected" From 590ce58e771cfd41691553caf1dd9050631ecc08 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 15 Oct 2024 15:23:41 -0400 Subject: [PATCH 174/238] adding json body parameter support --- bbot/core/helpers/diff.py | 6 +++ bbot/core/helpers/helper.py | 2 + bbot/core/helpers/misc.py | 1 + bbot/modules/internal/excavate.py | 50 +++++++++++++++++++ bbot/modules/lightfuzz_submodules/base.py | 45 ++++++++++++++--- .../module_tests/test_module_excavate.py | 49 +++++++++++++++++- 6 files changed, 146 insertions(+), 7 deletions(-) diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py index 138d73ac0..2827ac8c4 100644 --- a/bbot/core/helpers/diff.py +++ b/bbot/core/helpers/diff.py @@ -15,6 +15,7 @@ def __init__( parent_helper, method="GET", data=None, + json=None, allow_redirects=False, include_cache_buster=True, headers=None, @@ -26,6 +27,7 @@ def __init__( self.include_cache_buster = include_cache_buster self.method = method self.data = data + self.json = json self.allow_redirects = allow_redirects self._baselined = False self.headers = headers @@ -53,6 +55,7 @@ async def _baseline(self): follow_redirects=self.allow_redirects, method=self.method, data=self.data, + json=self.json, headers=self.headers, cookies=self.cookies, retries=2, @@ -76,6 +79,7 @@ async def _baseline(self): follow_redirects=self.allow_redirects, method=self.method, data=self.data, + json=self.json, retries=2, timeout=self.timeout, ) @@ -166,6 +170,7 @@ async def compare( check_reflection=False, method="GET", data=None, + json=None, allow_redirects=False, timeout=None, ): @@ -196,6 +201,7 @@ async def compare( follow_redirects=allow_redirects, method=method, data=data, + json=json, timeout=timeout, ) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index 1c31b81ae..a147293c1 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -128,6 +128,7 @@ def http_compare( cookies=None, method="GET", data=None, + json=None, timeout=10, ): return HttpCompare( @@ -140,6 +141,7 @@ def http_compare( timeout=timeout, method=method, data=data, + json=json, ) def temp_filename(self, extension=None): diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 6c8733eed..a795fa0c4 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -920,6 +920,7 @@ def extract_params_xml(xml_data, compare_mode="getparam"): "getparam": set(chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="), "postparam": set(chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="), "cookie": set(chr(c) for c in range(33, 127) if chr(c) not in '()<>@,;:"/[]?={} \t'), + "bodyjson": set(chr(c) for c in range(33, 127) if chr(c) not in ":/?#[]@!$&'()*+,;="), } diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 6316a7361..ac1953fa7 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -1,3 +1,4 @@ +import ast import yara import json import html @@ -431,6 +432,55 @@ def extract(self): query_strings_dict, parameter_name ) + class AjaxJquery(ParameterExtractorRule): + name = "JQuery Extractor" + discovery_regex = r"/\$\.ajax\(\{[^\<$\$]*\}\)/s nocase" + extraction_regex = None + output_type = "BODYJSON" + ajax_content_regexes = { + "url": r"url\s*:\s*['\"](.*?)['\"]", + "type": r"type\s*:\s*['\"](.*?)['\"]", + "content_type": r"contentType\s*:\s*['\"](.*?)['\"]", + "data": r"data:.*(\{[^}]*\})", + } + + def extract(self): + # Iterate through each regex in ajax_content_regexes + extracted_values = {} + for key, pattern in self.ajax_content_regexes.items(): + match = re.search(pattern, self.result) + if match: + # Store the matched value in the dictionary + extracted_values[key] = match.group(1) + + # check to see if the format is defined as JSON + if "content_type" in extracted_values.keys(): + if extracted_values["content_type"] == "application/json": + + # If we cant figure out the parameter names, there is no point in continuing + if "data" in extracted_values.keys(): + + if "url" in extracted_values.keys(): + form_url = extracted_values["url"] + else: + form_url = None + + form_parameters = {} + try: + s = extracted_values["data"] + s = re.sub(r"(\w+)\s*:", r'"\1":', s) # Quote keys + s = re.sub(r":\s*(\w+)", r': "\1"', s) # Quote values if they are unquoted + data = json.loads(s) + except (ValueError, SyntaxError): + return None + for p in data.keys(): + form_parameters[p] = None + + for parameter_name in form_parameters: + yield "BODYJSON", parameter_name, None, form_url, _exclude_key( + form_parameters, parameter_name + ) + class GetForm(ParameterExtractorRule): name = "GET Form" discovery_regex = r'/]*\bmethod=["\']?get["\']?[^>]*>.*<\/form>/s nocase' diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index b0654a075..e781d1a45 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -12,7 +12,7 @@ def additional_params_process(self, additional_params, additional_params_populat return additional_params new_additional_params = {} for k, v in additional_params.items(): - if v == "": + if v == "" or v == None: new_additional_params[k] = self.lightfuzz.helpers.rand_string(8, numeric_only=True) else: new_additional_params[k] = v @@ -66,10 +66,21 @@ def compare_baseline( http_compare = self.lightfuzz.helpers.http_compare( self.event.data["url"], method="POST", include_cache_buster=False, data=data, cookies=cookies ) + elif event_type == "BODYJSON": + data = {self.event.data["name"]: f"{probe}"} + if self.event.data["additional_params"] is not None: + data.update( + self.additional_params_process( + self.event.data["additional_params"], additional_params_populate_empty + ) + ) + http_compare = self.lightfuzz.helpers.http_compare( + self.event.data["url"], method="POST", include_cache_buster=False, json=data, cookies=cookies + ) return http_compare async def baseline_probe(self, cookies): - if self.event.data.get("eventtype") == "POSTPARAM": + if self.event.data.get("eventtype") in ["POSTPARAM", "BODYJSON"]: method = "POST" else: method = "GET" @@ -121,6 +132,13 @@ async def compare_probe( compare_result = await http_compare.compare( self.event.data["url"], method="POST", data=data, cookies=cookies ) + elif event_type == "BODYJSON": + data = {self.event.data["name"]: f"{probe}"} + if additional_params: + data.update(self.additional_params_process(additional_params, additional_params_populate_empty)) + compare_result = await http_compare.compare( + self.event.data["url"], method="POST", json=data, cookies=cookies + ) return compare_result async def standard_probe( @@ -154,6 +172,10 @@ async def standard_probe( headers = {self.event.data["name"]: probe} else: headers = {} + + data = None + json_data = None + if event_type == "POSTPARAM": method = "POST" data = {self.event.data["name"]: probe} @@ -163,14 +185,24 @@ async def standard_probe( self.event.data["additional_params"], additional_params_populate_empty ) ) - else: - data = {} + elif event_type == "BODYJSON": + method = "POST" + json_data = {self.event.data["name"]: probe} + if self.event.data["additional_params"] is not None: + json_data.update( + self.additional_params_process( + self.event.data["additional_params"], additional_params_populate_empty + ) + ) + + # Pass json_data if BODYJSON, otherwise pass data self.lightfuzz.debug(f"standard_probe requested URL: [{url}]") return await self.lightfuzz.helpers.request( method=method, cookies=cookies, headers=headers, data=data, + json=json_data, url=url, allow_redirects=False, retries=0, @@ -187,10 +219,11 @@ def metadata(self): return metadata_string def probe_value_incoming(self, populate_empty=True): - - probe_value = str(self.event.data.get("original_value", "")) + probe_value = self.event.data.get("original_value", "") if (probe_value is None or len(probe_value) == 0) and populate_empty == True: probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True) + if not isinstance(probe_value, str): + probe_value = str(probe_value) self.lightfuzz.debug(f"probe_value_incoming (before modification): {probe_value}") envelopes_instance = getattr(self.event, "envelopes", None) probe_value = envelopes_instance.remove_envelopes(probe_value) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 5099cfe5b..ad1b4aeda 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -709,11 +709,58 @@ def check(self, module_test, events): excavate_getparam_extraction = False for e in events: if e.type == "WEB_PARAMETER": - if "HTTP Extracted Parameter [novalue] (GET Form Submodule)": + if "HTTP Extracted Parameter [novalue] (GET Form Submodule)" in e.data["description"]: excavate_getparam_extraction = True assert excavate_getparam_extraction, "Excavate failed to extract web parameter" +class TestExcavateParameterExtraction_jqueryjsonajax(ModuleTestBase): + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["httpx", "excavate", "hunt"] + jsonajax_extract_html = """ + + " result = await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") - if result == False: + if result is False: in_javasscript_escape_probe = rf"a\';zzzzz({random_string})\\" in_javasscript_escape_match = rf"a\\';zzzzz({random_string})\\" await self.check_probe( diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index e1ee1e6d8..2b5732de2 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -794,7 +794,7 @@ def check(self, module_test, events): if e.type == "WEB_PARAMETER": if ( "HTTP Extracted Parameter [username] (JQuery Extractor Submodule)" == e.data["description"] - and e.data["original_value"] == None + and e.data["original_value"] is None ): excavate_ajaxpost_extraction = True assert excavate_ajaxpost_extraction, "Excavate failed to extract web parameter" @@ -1334,6 +1334,6 @@ def check(self, module_test, events): if e.data["name"] == "TS0113CC91": found_third_cookie = True - assert found_first_cookie == True - assert found_second_cookie == False - assert found_third_cookie == False + assert found_first_cookie is True + assert found_second_cookie is False + assert found_third_cookie is False diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 72d607c0c..a74ea74f5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -42,7 +42,7 @@ def request_handler(self, request): if "&" in value: value = value.split("&")[0] - block = f""" + block = """ @@ -581,7 +581,7 @@ def request_handler(self, request): """ - sql_block_error = f""" + sql_block_error = """

Found error in SQL query

@@ -652,7 +652,7 @@ def request_handler(self, request):
""" - sql_block_error = f""" + sql_block_error = """

Found error in SQL query

@@ -767,7 +767,7 @@ def request_handler(self, request):

test: {header_value}

""" - header_block_error = f""" + header_block_error = """

placeholder

Error!

@@ -838,7 +838,7 @@ def request_handler(self, request): """ - header_block_error = f""" + header_block_error = """

placeholder

Error!

@@ -885,7 +885,7 @@ def request_handler(self, request): if "&" in value: value = value.split("&")[0] - sql_block = f""" + sql_block = """

0 search results found

@@ -1328,7 +1328,7 @@ def request_handler(self, request):
""" - crypto_block = f""" + crypto_block = """

Access Denied!

diff --git a/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py b/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py index 812046a66..f1725a5ec 100644 --- a/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py +++ b/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py @@ -10,7 +10,7 @@ class TestReflected_parameters_fromexcavate(ModuleTestBase): modules_overrides = ["httpx", "reflected_parameters", "excavate"] def request_handler(self, request): - normal_block = f'foo' + normal_block = 'foo' qs = str(request.query_string.decode()) if "reflected=" in qs: value = qs.split("=")[1] From 0898c31f2ebe1cadf54ce48d2d49f7c584173209 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 27 Nov 2024 11:38:31 -0500 Subject: [PATCH 213/238] preset whitespace --- bbot/presets/web/lightfuzz-max.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/presets/web/lightfuzz-max.yml b/bbot/presets/web/lightfuzz-max.yml index b05a5ee79..8bb5f51de 100644 --- a/bbot/presets/web/lightfuzz-max.yml +++ b/bbot/presets/web/lightfuzz-max.yml @@ -21,6 +21,6 @@ config: lightfuzz: force_common_headers: True enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss] - disable_post: False + disable_post: False excavate: retain_querystring: True \ No newline at end of file From e0e675685d9fb9c2e727275096fb6fe36fd71226 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 27 Nov 2024 13:41:58 -0500 Subject: [PATCH 214/238] regex tweaks --- bbot/core/helpers/regexes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 4b39a6a42..d0a900a0d 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -120,12 +120,12 @@ # For use with excavate parameters extractor input_tag_regex = re.compile( - r"]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([%\._=+\/\w]*)[\"\']?[^>]*?>" + r"]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([%\-\._=+\/\w]*)[\"\']?[^>]*?>" ) input_tag_regex2 = re.compile( - r"]*?value=[\"\']?([%\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?>" + r"]*?value=[\"\']?([\-%\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>" ) -input_tag_novalue_regex = re.compile(r"]*\bvalue=)[^>]*?name=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>") +input_tag_novalue_regex = re.compile(r"]*\bvalue=)[^>]*?name=[\"\']?([\-\._=+\/\w]*)[\"\']?[^>]*?>") # jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") # jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}") # jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}") @@ -156,10 +156,10 @@ ) select_tag_regex = re.compile( - r"]+?name=[\"\']?(\w+)[\"\']?[^>]*>(?:\s*]*?value=[\"\'](\w*)[\"\']?[^>]*>)?" + r"]+?name=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>(?:\s*]*?value=[\"\']?([_\.\-\w]*)[\"\']?[^>]*>)?" ) textarea_tag_regex = re.compile( - r']*\bname=["\']?(\w+)["\']?[^>]*>(.*?)', re.IGNORECASE | re.DOTALL + r']*\bname=["\']?([_\-\.\w]+)["\']?[^>]*>(.*?)', re.IGNORECASE | re.DOTALL ) tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>") From 2763487e5aca8d049d96cafa341f0ea78471e53d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 27 Nov 2024 13:46:18 -0500 Subject: [PATCH 215/238] adding test for select tags --- .../module_tests/test_module_excavate.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 2b5732de2..447b35f18 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -450,7 +450,6 @@ class TestExcavateParameterExtraction(TestExcavate): $.post("/test", {jquerypost: "value2"}); -

Simple GET Form

Use the form below to submit a GET request:

@@ -476,7 +475,11 @@ class TestExcavateParameterExtraction(TestExcavate):

Links

href img - + """ @@ -497,6 +500,7 @@ def check(self, module_test, events): found_form_generic_original_value = False found_htmltags_a = False found_htmltags_img = False + found_select_noquotes = False for e in events: @@ -537,6 +541,11 @@ def check(self, module_test, events): if "fit" in e.data["additional_params"].keys(): found_htmltags_img = True + if e.data["description"] == "HTTP Extracted Parameter [blog-post-author-display] (POST Form Submodule)": + if e.data["original_value"] == "user.name": + if "csrf" in e.data["additional_params"].keys(): + found_select_noquotes = True + assert found_jquery_get, "Did not extract Jquery GET parameters" assert found_jquery_post, "Did not extract Jquery POST parameters" assert found_form_get, "Did not extract Form GET parameters" @@ -549,7 +558,7 @@ def check(self, module_test, events): assert found_form_generic_original_value, "Did not extract Form (Generic) parameter original_value" assert found_htmltags_a, "Did not extract parameter(s) from a-tag" assert found_htmltags_img, "Did not extract parameter(s) from img-tag" - + assert found_select_noquotes, "Did not extract parameter(s) from select-tag" class TestExcavateParameterExtraction_postformnoaction(ModuleTestBase): From 0fef49ca5c3f10bdcb0f845dee3b8d34b7e275a8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 27 Nov 2024 18:09:13 -0500 Subject: [PATCH 216/238] tests passing --- bbot/core/event/base.py | 407 ++---------------- bbot/core/helpers/misc.py | 12 + bbot/core/helpers/web/client.py | 1 - bbot/core/helpers/web/engine.py | 2 - bbot/core/helpers/web/envelopes.py | 352 +++++++++++++++ bbot/modules/base.py | 5 +- bbot/modules/internal/cloudcheck.py | 4 +- bbot/modules/internal/dnsresolve.py | 4 +- bbot/modules/internal/excavate.py | 23 +- bbot/modules/lightfuzz.py | 12 +- bbot/modules/lightfuzz_submodules/base.py | 38 +- bbot/modules/lightfuzz_submodules/cmdi.py | 3 - bbot/modules/lightfuzz_submodules/crypto.py | 9 +- bbot/modules/lightfuzz_submodules/path.py | 1 - bbot/modules/lightfuzz_submodules/sqli.py | 1 - bbot/modules/report/asn.py | 9 +- bbot/scanner/preset/args.py | 4 +- bbot/scanner/preset/preset.py | 2 +- bbot/test/test_step_1/test__module__tests.py | 1 - bbot/test/test_step_1/test_bbot_fastapi.py | 3 - bbot/test/test_step_1/test_bloom_filter.py | 1 - bbot/test/test_step_1/test_dns.py | 3 - bbot/test/test_step_1/test_engine.py | 2 - bbot/test/test_step_1/test_events.py | 2 - bbot/test/test_step_1/test_helpers.py | 1 - bbot/test/test_step_1/test_presets.py | 7 +- bbot/test/test_step_1/test_target.py | 1 - bbot/test/test_step_1/test_web.py | 2 - bbot/test/test_step_1/test_web_envelopes.py | 339 +++++++++++++++ .../module_tests/test_module_baddns_direct.py | 6 +- .../module_tests/test_module_excavate.py | 11 - .../module_tests/test_module_gowitness.py | 4 +- .../module_tests/test_module_hunt.py | 1 - .../module_tests/test_module_lightfuzz.py | 322 ++++++-------- .../module_tests/test_module_ntlm.py | 3 +- .../test_module_reflected_parameters.py | 2 - .../module_tests/test_module_speculate.py | 4 +- 37 files changed, 930 insertions(+), 674 deletions(-) create mode 100644 bbot/core/helpers/web/envelopes.py create mode 100644 bbot/test/test_step_1/test_web_envelopes.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index d3cf2c2ba..fc2a35ea2 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1,24 +1,20 @@ -from collections import defaultdict import io import re import uuid import json import base64 -import asyncio import logging import tarfile -import binascii import datetime import ipaddress import traceback -import xml.etree.ElementTree as ET -from copy import copy, deepcopy from pathlib import Path from typing import Optional +from copy import copy, deepcopy from contextlib import suppress from radixtarget import RadixTarget -from urllib.parse import urljoin, parse_qs, unquote, quote +from urllib.parse import urljoin, parse_qs from pydantic import BaseModel, field_validator @@ -44,6 +40,7 @@ validators, get_file_extension, ) +from bbot.core.helpers.web.envelopes import BaseEnvelope log = logging.getLogger("bbot.core.event") @@ -593,6 +590,10 @@ def parent(self, parent): elif not self._dummy: log.warning(f"Tried to set invalid parent on {self}: (got: {parent})") + @property + def children(self): + return [] + @property def parent_id(self): parent_id = getattr(self.get_parent(), "id", None) @@ -648,15 +649,10 @@ def get_parents(self, omit=False, include_self=False): return parents def clone(self): - # Create a shallow copy of the event first cloned_event = copy(self) - - # Handle attributes that need deep copying manually - setattr(cloned_event, "envelopes", deepcopy(self.envelopes)) - # Re-assign a new UUID - cloned_event.uuid = uuid.uuid4() + cloned_event._uuid = uuid.uuid4() return cloned_event def _host(self): @@ -1329,382 +1325,39 @@ class URL_HINT(URL_UNVERIFIED): class WEB_PARAMETER(DictHostEvent): - @property - def uuid(self): - return self._uuid - - @uuid.setter - def uuid(self, value): - self._uuid = value - - class ParameterEnvelopes: - - @staticmethod - def preprocess_base64(base64_str): - return base64.b64decode(base64_str).decode() - - @staticmethod - def postprocess_base64(string): - return base64.b64encode(string.encode()).decode() - - @staticmethod - def preprocess_hex(hex_str): - return bytes.fromhex(hex_str).decode() - - @staticmethod - def postprocess_hex(string): - return string.encode().hex() - - @staticmethod - def preprocess_urlencoded(url_encoded_str): - return unquote(url_encoded_str) - - @staticmethod - def postprocess_urlencoded(string): - return quote(string) - - @staticmethod - def is_ascii_printable(s): - return all(32 <= ord(char) < 127 for char in s) - - # Converts XML ElementTree to a JSON-like dictionary - def xml_to_dict(self, elem): - """ - Convert XML ElementTree to a dictionary recursively. - """ - d = {elem.tag: {} if elem.attrib else None} - children = list(elem) - if children: - dd = defaultdict(list) - for dc in map(self.xml_to_dict, children): - for k, v in dc.items(): - dd[k].append(v) - d = {elem.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}} - if elem.attrib: - d[elem.tag].update(("@" + k, v) for k, v in elem.attrib.items()) - if elem.text: - text = elem.text.strip() - if children or elem.attrib: - if text: - d[elem.tag]["#text"] = text - else: - d[elem.tag] = text - return d - - def dict_to_xml(self, d): - """ - Converts a dictionary to an XML string without adding an extra root node. - Assumes the dictionary was originally an XML structure. - """ - if not isinstance(d, dict) or len(d) != 1: - raise ValueError("Expected a dictionary with a single root element.") - - # Get the root element directly from the dict keys - root_tag = list(d.keys())[0] - root_element = ET.Element(root_tag) - - # Recursive function to handle nested dicts - def _build_tree(element, subdict): - for key, value in subdict.items(): - if isinstance(value, dict): - # Nested element - sub_element = ET.SubElement(element, key) - _build_tree(sub_element, value) - else: - # Leaf element - sub_element = ET.SubElement(element, key) - sub_element.text = str(value) - - # Start building the tree - _build_tree(root_element, d[root_tag]) - - return ET.tostring(root_element, encoding="utf-8").decode("utf-8") - - preprocess_map = { - "base64": preprocess_base64, - "hex": preprocess_hex, - "url-encoded": preprocess_urlencoded, - } - postprocess_map = { - "base64": postprocess_base64, - "hex": postprocess_hex, - "url-encoded": postprocess_urlencoded, - } - - # Format-specific functions for isolating and updating parameters - format_isolate_map = { - "json": lambda self: self.isolate_parameter(), - "xml": lambda self: self.isolate_parameter(), - } - format_update_map = { - "json": lambda self, value: self.update_json_parameter(value), - "xml": lambda self, value: self.update_xml_parameter(value), # Placeholder - } - - def initialize_value(self, value=None): - self.envelopes, end_format_dict = self.recurse_envelopes(value) - if self.envelopes: - log.debug(f"Discovered the following envelopes: [{','.join(self.envelopes)}]") - - if end_format_dict is not None: - self.end_format_type = list(end_format_dict.keys())[0] - log.debug(f"Identified the following end format: [{self.end_format_type}]") - self.end_format_data = list(end_format_dict.values())[0] - else: - self.end_format_type = None - self.end_format_data = None - self.end_format_subparameter = None - - def remove_envelopes(self, value): - """ - Remove envelopes from the value, processing each envelope in the order it was applied. - If the final format is present, trigger the appropriate handler (e.g., for JSON). - """ - # Apply the preprocess functions in the order the envelopes were applied - for env in self.envelopes: - func = self.preprocess_map.get(env) - if func: - # python3.9 compatibility hack - if isinstance(func, staticmethod): - func = func.__get__(None, self.__class__) # Unwrap staticmethod - value = func(value) - - # Dynamically select the appropriate isolate function based on the final format - isolate_func = self.format_isolate_map.get(self.end_format_type) - if isolate_func: - return isolate_func(self) - return value - - def add_envelopes(self, value): - """ - Add envelopes back to the value, processing in reverse order. - If the final format is present, trigger the appropriate handler (e.g., for JSON). - """ - # Dynamically select the appropriate update function based on the final format - update_func = self.format_update_map.get(self.end_format_type) - if update_func: - # python3.9 compatibility hack - if isinstance(update_func, staticmethod): - update_func = update_func.__get__(None, self.__class__) - value = update_func(self, value) - - # Apply the envelopes in reverse order - for env in self.envelopes[::-1]: - func = self.postprocess_map.get(env) - if func: - # python3.9 compatibility hack - if isinstance(func, staticmethod): - func = func.__get__(None, self.__class__) - value = func(value) - return value - - def recurse_envelopes(self, value, envelopes=None, end_format=None): - if envelopes is None: - envelopes = [] - log.debug( - f"Starting envelope recurse with value: [{value}], current envelopes: [{', '.join(envelopes)}], current end format: {end_format}" - ) - - if value is None or value == "" or isinstance(value, int): - return envelopes, end_format - - # Try URL decoding - try: - decoded_url = unquote(value) - if decoded_url != value and self.is_ascii_printable(decoded_url): - envelopes.append("url-encoded") - envelopes, end_format_dict = self.recurse_envelopes(decoded_url, envelopes) - return envelopes, end_format_dict - except Exception: - pass # Not valid URL encoding - - # Try base64 decoding - try: - decoded_base64 = base64.b64decode(value).decode() - if self.is_ascii_printable(decoded_base64): - envelopes.append("base64") - envelopes, end_format_dict = self.recurse_envelopes(decoded_base64, envelopes) - return envelopes, end_format_dict - except (binascii.Error, UnicodeDecodeError, ValueError): - pass # Not valid base64 - - # Try hex decoding - try: - decoded_hex = bytes.fromhex(value).decode("utf-8") - if self.is_ascii_printable(decoded_hex): - envelopes.append("hex") - envelopes, end_format_dict = self.recurse_envelopes(decoded_hex, envelopes) - return envelopes, end_format_dict - except (ValueError, UnicodeDecodeError): - pass # Not valid hex - - # Try JSON parsing - try: - decoded_json = json.loads(value) - if isinstance(decoded_json, dict): - return envelopes, {"json": decoded_json} - except json.JSONDecodeError: - pass # Not valid JSON - - # Try XML parsing - try: - decoded_xml = ET.fromstring(value) - # Pass 'decoded_xml' to 'xml_to_dict' - xml_dict = self.xml_to_dict(decoded_xml) # Pass decoded XML as the 'elem' argument - return envelopes, {"xml": xml_dict} # Store as JSON-like dict, not XML - except ET.ParseError: - pass # Not valid XML - - return envelopes, end_format - - def isolate_parameter(self): - """ - Isolate the specified subparameter from the data structure (JSON/XML). - The subparameter is accessed using dot notation for nested keys. - """ - if self.end_format_data and self.end_format_subparameter: - # Split the dot notation string into keys - keys = self.end_format_subparameter.split(".") - - # Traverse the nested structure using the keys - subparameter_value = self.end_format_data - for key in keys: - if isinstance(subparameter_value, dict): - subparameter_value = subparameter_value.get(key) - else: - # If the structure is broken (not a dict), return None - return None - - return subparameter_value - - return None - - def update_json_parameter(self, new_value): - """ - Update the specified subparameter in the JSON structure and rebuild it. - """ - # Work with a copy to avoid modifying the original `end_format_data` - end_format_data_copy = deepcopy(self.end_format_data) - - if end_format_data_copy: - end_format_data_copy[self.end_format_subparameter] = new_value - return json.dumps(end_format_data_copy) - return new_value - - def update_xml_parameter(self, new_value): - """ - Convert the JSON-like structure back into an XML string after updating the specific parameter. - """ - if self.end_format_data and self.end_format_subparameter: - # Split the dot notation into keys - keys = self.end_format_subparameter.split(".") - - # Traverse the nested dictionary using the keys to find the target subparameter - current_data = self.end_format_data - for key in keys[:-1]: # Traverse up to the second-to-last key - current_data = current_data.get(key, {}) - - # Update the target subparameter with the new value - if isinstance(current_data, dict): - current_data[keys[-1]] = new_value - - # Convert the JSON-like dict back to an XML string - return self.dict_to_xml(self.end_format_data) - - return new_value - - def to_dict(self): - return { - "envelopes": self.envelopes, - "end_format_type": self.end_format_type, - "end_format_data": self.end_format_data, - "end_format_subparameter": self.end_format_subparameter, - } - - def __getstate__(self): - return self.to_dict() - - def __str__(self): - return f"ParameterEnvelopes(envelopes={self.envelopes}, end_format_type={self.end_format_type}, end_format_data={self.end_format_data}, end_format_subparameter={self.end_format_subparameter})" - - __repr__ = __str__ - - @classmethod - def from_dict(cls, data): - instance = cls() - instance.envelopes = data.get("envelopes", []) - instance.end_format_type = data.get("end_format_type") - instance.end_format_data = data.get("end_format_data") - instance.end_format_subparameter = data.get("end_format_subparameter") - return instance + def children(self): + # if we have any subparams, raise a new WEB_PARAMETER for each one + children = [] + envelopes = getattr(self, "envelopes", None) + if envelopes is not None: + subparams = list(self.envelopes.get_subparams()) + + if envelopes.selected_subparam is None: + for subparam, _ in subparams: + clone = self.clone() + clone.envelopes = deepcopy(envelopes) + clone.envelopes.selected_subparam = subparam + clone.parent = self + children.append(clone) + return children def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if "original_value" in self.data.keys(): - parameterEnvelope_instance = self.ParameterEnvelopes() - parameterEnvelope_instance.initialize_value(self.data["original_value"]) - setattr(self, "envelopes", parameterEnvelope_instance) - - envelopes = getattr(self, "envelopes", None) - if ( - envelopes is not None - and getattr(envelopes, "end_format_type", None) is not None - and getattr(envelopes, "end_format_data", None) - ): - end_format_data = envelopes.end_format_data - - def extract_keys_with_values(data, parent_key=""): - """ - Recursively extract all keys from nested dictionaries that have values (non-empty). - Construct a path-like structure with dot notation (e.g., 'find.search'). - """ - keys = [] - if isinstance(data, dict): - for key, value in data.items(): - # Construct the full key path using dot notation - full_key = f"{parent_key}.{key}" if parent_key else key - - # Only add keys that have non-empty values - if value: - if isinstance(value, dict): - # Recursively check nested dictionaries - keys.extend(extract_keys_with_values(value, full_key)) - else: - # Add the key if it has a non-empty value - keys.append(full_key) - return keys - - # Extract all keys that have non-empty values - end_format_data_keys = extract_keys_with_values(end_format_data) - # If there are keys, assign the first key to end_format_subparameter - if end_format_data_keys: - - # Assign the first key to end_format_subparameter - setattr(envelopes, "end_format_subparameter", end_format_data_keys[0]) - setattr(envelopes, "end_format_subparameter", end_format_data_keys[0]) - - # Iterate through the remaining keys, starting from the second one - for p in end_format_data_keys[1:]: - log.debug(f"generating copy of event for subparameter {p} of type {envelopes.end_format_type}") - - # Make a copy of the current event data - cloned_event = self.clone() - cloned_envelopes = getattr(cloned_event, "envelopes") - cloned_envelopes.end_format_subparameter = p - asyncio.run_coroutine_threadsafe( - self.module.emit_event(cloned_event), asyncio.get_event_loop() - ) + original_value = self.data.get("original_value", None) + if original_value is not None: + envelopes = BaseEnvelope.detect(original_value) + setattr(self, "envelopes", envelopes) def _data_id(self): # dedupe by url:name:param_type url = self.data.get("url", "") name = self.data.get("name", "") param_type = self.data.get("type", "") - envelopes = getattr(self, "envelopes", None) - subparameter = getattr(envelopes, "end_format_subparameter", "") if envelopes else "" + envelopes = getattr(self, "envelopes", "") + subparam = getattr(envelopes, "selected_subparam", "") - return f"{url}:{name}:{param_type}:{subparameter}" + return f"{url}:{name}:{param_type}:{subparam}" def _outgoing_dedup_hash(self, event): return hash( diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 7c6c8a073..73bd45f30 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2867,3 +2867,15 @@ def clean_requirement(req_string): dist = distribution("bbot") return [clean_requirement(r) for r in dist.requires] + + +printable_chars = set(string.printable) + + +def is_printable(s): + """ + Check if a string is printable + """ + if not isinstance(s, str): + raise ValueError(f"Expected a string, got {type(s)}") + return set(s) <= printable_chars diff --git a/bbot/core/helpers/web/client.py b/bbot/core/helpers/web/client.py index 83154e5ae..49ddf532b 100644 --- a/bbot/core/helpers/web/client.py +++ b/bbot/core/helpers/web/client.py @@ -85,7 +85,6 @@ def __init__(self, *args, **kwargs): self._cookies = DummyCookies() def build_request(self, *args, **kwargs): - if args: url = args[0] kwargs["url"] = url diff --git a/bbot/core/helpers/web/engine.py b/bbot/core/helpers/web/engine.py index e0f63cb05..8ffdbe966 100644 --- a/bbot/core/helpers/web/engine.py +++ b/bbot/core/helpers/web/engine.py @@ -50,7 +50,6 @@ def AsyncClient(self, *args, **kwargs): return client async def request(self, *args, **kwargs): - raise_error = kwargs.pop("raise_error", False) # TODO: use this cache_for = kwargs.pop("cache_for", None) # noqa @@ -75,7 +74,6 @@ async def request(self, *args, **kwargs): client_kwargs = {} for k in list(kwargs): if k in self.client_only_options: - v = kwargs.pop(k) client_kwargs[k] = v diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py new file mode 100644 index 000000000..e84be51ba --- /dev/null +++ b/bbot/core/helpers/web/envelopes.py @@ -0,0 +1,352 @@ +import json +import base64 +import binascii +import xmltodict +from contextlib import suppress +from urllib.parse import unquote, quote +from xml.parsers.expat import ExpatError + +from bbot.core.helpers.misc import is_printable + + +# TODO: This logic is perfect for extracting params. We should expand it outwards to other higher-level envelopes: +# - QueryStringEnvelope +# - MultipartFormEnvelope +# - HeaderEnvelope +# - CookieEnvelope +# +# Once we start ingesting HTTP_REQUEST events, this will make them instantly fuzzable + + +class EnvelopeChildTracker(type): + """ + Keeps track of all the child envelope classes + """ + + children = [] + + def __new__(mcs, name, bases, class_dict): + # Create the class + cls = super().__new__(mcs, name, bases, class_dict) + # Don't register the base class itself + if bases and not name.startswith("Base"): # Only register if it has base classes (i.e., is a child) + EnvelopeChildTracker.children.append(cls) + EnvelopeChildTracker.children.sort(key=lambda x: x.priority) + return cls + + +class BaseEnvelope(metaclass=EnvelopeChildTracker): + __slots__ = ["subparams", "selected_subparam", "singleton"] + + # determines the order of the envelope detection + priority = 5 + # whether the envelope is the final format, e.g. raw text/binary + end_format = False + ignore_exceptions = (Exception,) + envelope_classes = EnvelopeChildTracker.children + # transparent envelopes (i.e. TextEnvelope) are not counted as envelopes or included in the finding descriptions + transparent = False + + def __init__(self, s): + unpacked_data = self.unpack(s) + + if self.end_format: + inner_envelope = unpacked_data + else: + inner_envelope = self.detect(unpacked_data) + + self.selected_subparam = None + # if we have subparams, our inner envelope will be a dictionary + if isinstance(inner_envelope, dict): + self.subparams = inner_envelope + self.singleton = False + # otherwise if we just have one value, we make a dictionary with a default key + else: + self.subparams = {"__default__": inner_envelope} + self.singleton = True + + @property + def final_envelope(self): + try: + return self.unpacked_data(recursive=False).final_envelope + except AttributeError: + return self + + def pack(self, data=None): + if data is None: + data = self.unpacked_data(recursive=False) + with suppress(AttributeError): + data = data.pack() + return self._pack(data) + + def unpack(self, s): + return self._unpack(s) + + def _pack(self, s): + """ + Encodes the string using the class's unique encoder (adds the outer envelope) + """ + raise NotImplementedError("Envelope.pack() must be implemented") + + def _unpack(self, s): + """ + Decodes the string using the class's unique encoder (removes the outer envelope) + """ + raise NotImplementedError("Envelope.unpack() must be implemented") + + def unpacked_data(self, recursive=True): + try: + unpacked = self.subparams["__default__"] + if recursive: + with suppress(AttributeError): + return unpacked.unpacked_data(recursive=recursive) + return unpacked + except KeyError: + return self.subparams + + @classmethod + def detect(cls, s): + """ + Detects the type of envelope used to encode the packed_data + """ + if not isinstance(s, str): + raise ValueError(f"Invalid data passed to detect(): {s} ({type(s)})") + # if the value is empty, we just return the text envelope + if not s.strip(): + return TextEnvelope(s) + for envelope_class in cls.envelope_classes: + with suppress(*envelope_class.ignore_exceptions): + envelope = envelope_class(s) + if envelope is not False: + return envelope + del envelope + raise Exception("No envelope detected") + + def get_subparams(self, key=None, data=None, recursive=True): + if data is None: + data = self.unpacked_data(recursive=recursive) + if key is None: + key = [] + + if isinstance(data, dict): + for k, v in data.items(): + full_key = key + [k] + if isinstance(v, dict): + yield from self.get_subparams(full_key, v) + else: + yield full_key, v + else: + yield [], data + + def get_subparam(self, key=None, recursive=True): + if key is None: + key = self.selected_subparam + envelope = self + if recursive: + envelope = self.final_envelope + data = envelope.unpacked_data(recursive=False) + if key is None: + if envelope.singleton: + key = [] + else: + raise ValueError("No subparam selected") + else: + for segment in key: + data = data[segment] + return data + + def set_subparam(self, key=None, value=None, recursive=True): + envelope = self + if recursive: + envelope = self.final_envelope + + # if there's only one value to set, we can just set it directly + if envelope.singleton: + envelope.subparams["__default__"] = value + return + + # if key isn't specified, use the selected subparam + if key is None: + key = self.selected_subparam + if key is None: + raise ValueError(f"{self} -> {envelope}: No subparam selected") + + data = envelope.unpacked_data(recursive=False) + for segment in key[:-1]: + data = data[segment] + data[key[-1]] = value + + @property + def name(self): + return self.__class__.__name__ + + @property + def num_envelopes(self): + num_envelopes = 0 if self.transparent else 1 + if self.end_format: + return num_envelopes + for envelope in self.subparams.values(): + with suppress(AttributeError): + num_envelopes += envelope.num_envelopes + return num_envelopes + + @property + def summary(self): + if self.transparent: + return "" + self_string = f"{self.name}" + if self.selected_subparam: + self_string += f" [{self.selected_subparam}]" + with suppress(AttributeError): + child_envelope = self.unpacked_data(recursive=False) + child_summary = child_envelope.summary + if child_summary: + self_string += f" -> {child_summary}" + return self_string + + +class HexEnvelope(BaseEnvelope): + """ + Hexadecimal encoding + """ + + ignore_exceptions = (ValueError, UnicodeDecodeError) + + def _pack(self, s): + return s.encode().hex() + + def _unpack(self, s): + return bytes.fromhex(s).decode() + + +class B64Envelope(BaseEnvelope): + """ + Base64 encoding + """ + + ignore_exceptions = (binascii.Error, UnicodeDecodeError, ValueError) + + def unpack(self, s): + # it's easy to have a small value that accidentally decodes to base64 + if len(s) < 8 and not s.endswith("="): + raise ValueError("Data is too small to be sure") + return super().unpack(s) + + def _pack(self, s): + return base64.b64encode(s.encode()).decode() + + def _unpack(self, s): + return base64.b64decode(s).decode() + + +class URLEnvelope(BaseEnvelope): + """ + URL encoding + """ + + def unpack(self, s): + unpacked = super().unpack(s) + if unpacked == s: + raise Exception("Data is not URL-encoded") + return unpacked + + def _pack(self, s): + return quote(s) + + def _unpack(self, s): + return unquote(s) + + +class TextEnvelope(BaseEnvelope): + """ + Text encoding + """ + + end_format = True + # lowest priority means text is the ultimate fallback + priority = 10 + transparent = True + + def _pack(self, s): + return s + + def _unpack(self, s): + if not is_printable(s): + raise Exception("Non-printable data detected in TextEnvelope") + return s + + +# class BinaryEnvelope(BaseEnvelope): +# """ +# Binary encoding +# """ +# end_format = True + +# def pack(self, s): +# return s + +# def unpack(self, s): +# if is_printable(s): +# raise Exception("Non-binary data detected in BinaryEnvelope") +# return s + + +class BaseSubparamEnvelope(BaseEnvelope): + """ + An envelope that contains subparameters + """ + + end_format = True + + # def get_subparam(self, dot_key=None): + # if dot_key is None: + # dot_key = self.subparam + # data = self.unpacked_data + # for key in dot_key.split("."): + # data = data[key] + # return data + + # def values(self): + # """ + # Returns the values of the unpacked data + + # { + # "key1": "value1", + # "key2": "value2" + # } + # """ + # if isinstance(self.unpacked_data, dict): + # return self.unpacked_data.values() + # return [self.unpacked_data] + + +class JSONEnvelope(BaseEnvelope): + """ + JSON encoding + """ + + end_format = True + priority = 8 + ignore_exceptions = (json.JSONDecodeError,) + + def _pack(self, s): + return json.dumps(s) + + def _unpack(self, s): + return json.loads(s) + + +class XMLEnvelope(BaseEnvelope): + """ + XML encoding + """ + + end_format = True + priority = 9 + ignore_exceptions = (ExpatError,) + + def _pack(self, s): + return xmltodict.unparse(s) + + def _unpack(self, s): + return xmltodict.parse(s) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 1fa151c33..4ab2da152 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -528,8 +528,9 @@ async def emit_event(self, *args, **kwargs): if v is not None: emit_kwargs[o] = v event = self.make_event(*args, **event_kwargs) - if event: - await self.queue_outgoing_event(event, **emit_kwargs) + children = event.children + for e in [event] + children: + await self.queue_outgoing_event(e, **emit_kwargs) return event async def _events_waiting(self, batch_size=None): diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 685d67f9d..86b6130d7 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -57,7 +57,9 @@ async def handle_event(self, event, **kwargs): for provider in self.helpers.cloud.providers.values(): provider_name = provider.name.lower() base_kwargs = { - "parent": event, "tags": [f"{provider.provider_type}-{provider_name}"], "_provider": provider_name + "parent": event, + "tags": [f"{provider.provider_type}-{provider_name}"], + "_provider": provider_name, } # loop through the provider's regex signatures, if any for event_type, sigs in provider.signatures.items(): diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 15facec56..bdca0ea5c 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -306,9 +306,7 @@ def get_dns_parent(self, event): @property def emit_raw_records(self): if self._emit_raw_records is None: - watching_raw_records = any( - "RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values() - ) + watching_raw_records = any("RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()) omitted_event_types = self.scan.config.get("omit_event_types", []) omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types self._emit_raw_records = watching_raw_records or not omit_raw_records diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index d9da92a08..78b19c857 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -460,10 +460,8 @@ def extract(self): # check to see if the format is defined as JSON if "content_type" in extracted_values.keys(): if extracted_values["content_type"] == "application/json": - # If we cant figure out the parameter names, there is no point in continuing if "data" in extracted_values.keys(): - if "url" in extracted_values.keys(): form_url = extracted_values["url"] else: @@ -481,8 +479,12 @@ def extract(self): form_parameters[p] = None for parameter_name in form_parameters: - yield "BODYJSON", parameter_name, None, form_url, _exclude_key( - form_parameters, parameter_name + yield ( + "BODYJSON", + parameter_name, + None, + form_url, + _exclude_key(form_parameters, parameter_name), ) class GetForm(ParameterExtractorRule): @@ -501,7 +503,6 @@ class GetForm(ParameterExtractorRule): def extract(self): forms = self.extraction_regex.findall(str(self.result)) for form_action, form_content in forms: - if not form_action or form_action == "#": form_action = None @@ -512,7 +513,6 @@ def extract(self): for form_content_regex_name, form_content_regex in self.form_content_regexes.items(): input_tags = form_content_regex.findall(form_content) if input_tags: - if form_content_regex_name == "input_tag_novalue_regex": form_parameters[input_tags[0]] = None @@ -525,11 +525,11 @@ def extract(self): for parameter_name, original_value in form_parameters.items(): yield ( - self.output_type, - parameter_name, - original_value, + self.output_type, + parameter_name, + original_value, form_action, - _exclude_key(form_parameters, parameter_name), + _exclude_key(form_parameters, parameter_name), ) class GetForm2(GetForm): @@ -993,6 +993,8 @@ async def setup(self): return True async def search(self, data, event, content_type, discovery_context="HTTP response"): + # TODO: replace this JSON/XML extraction with our lightfuzz envelope stuff + if not data: return None decoded_data = await self.helpers.re.recursive_decode(data) @@ -1084,7 +1086,6 @@ async def handle_event(self, event): # If parameter_extraction is enabled and we assigned custom headers, emit them as WEB_PARAMETER if self.parameter_extraction == True: - custom_cookies = self.scan.web_config.get("http_cookies", {}) for custom_cookie_name, custom_cookie_value in custom_cookies.items(): description = f"HTTP Extracted Parameter [{custom_cookie_name}] (Custom Cookie)" diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index b1fcd19b2..a28564ab8 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -130,11 +130,8 @@ async def run_submodule(self, submodule, event): event_data = {"host": str(event.host), "url": event.data["url"], "description": r["description"]} envelopes = getattr(event, "envelopes", None) - if envelopes and envelopes.envelopes: - envelope_summary = f'[{"->".join(envelopes.envelopes)}]' - if envelopes.end_format_type: - envelope_summary += f" Format: [{envelopes.end_format_type}] with subparameter [{envelopes.end_format_subparameter}])" - + envelope_summary = getattr(envelopes, "summary", None) + if envelope_summary: # Append the envelope summary to the description event_data["description"] += f" Envelopes: {envelope_summary}" @@ -147,10 +144,8 @@ async def run_submodule(self, submodule, event): ) async def handle_event(self, event): - if event.type == "URL": if self.config.get("force_common_headers", False) == False: - return False for h in self.common_headers: @@ -166,7 +161,6 @@ async def handle_event(self, event): await self.emit_event(data, "WEB_PARAMETER", event) elif event.type == "WEB_PARAMETER": - # check connectivity to url connectivity_test = await self.helpers.request(event.data["url"], timeout=10) @@ -199,5 +193,5 @@ async def finish(self): async def filter_event(self, event): if event.type == "WEB_PARAMETER" and self.disable_post and event.data["type"] == "POSTPARAM": - return False, "POST parameter disabled in lilghtfuzz module" + return False, "POST parameter disabled in lightfuzz module" return True diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index efe8f8e76..e031d1742 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -104,7 +104,6 @@ async def compare_probe( additional_params_override={}, speculative_mode="GETPARAM", ): - probe = self.probe_value_outgoing(probe) additional_params = copy.deepcopy(self.event.data.get("additional_params", {})) if additional_params_override: @@ -150,7 +149,6 @@ async def standard_probe( additional_params_populate_empty=False, speculative_mode="GETPARAM", ): - probe = self.probe_value_outgoing(probe) if event_type == "SPECULATIVE": @@ -209,7 +207,6 @@ async def standard_probe( ) def metadata(self): - metadata_string = f"Parameter: [{self.event.data['name']}] Parameter Type: [{self.event.data['type']}]" if self.event.data["original_value"] != "" and self.event.data["original_value"] != None: metadata_string += ( @@ -218,20 +215,31 @@ def metadata(self): return metadata_string def probe_value_incoming(self, populate_empty=True): - probe_value = self.event.data.get("original_value", "") - if (probe_value is None or len(str(probe_value)) == 0) and populate_empty == True: - probe_value = self.lightfuzz.helpers.rand_string(10, numeric_only=True) - self.lightfuzz.debug(f"probe_value_incoming (before modification): {probe_value}") - envelopes_instance = getattr(self.event, "envelopes", None) - probe_value = envelopes_instance.remove_envelopes(probe_value) - self.lightfuzz.debug(f"probe_value_incoming (after modification): {probe_value}") + envelopes = getattr(self.event, "envelopes", None) + probe_value = "" + if envelopes is not None: + probe_value = envelopes.get_subparam() + if not probe_value: + if populate_empty is True: + probe_value = self.lightfuzz.helpers.rand_string(10, numeric_only=True) + else: + probe_value = "" + self.lightfuzz.debug(f"probe_value_incoming (before unpacking): {probe_value}") + if envelopes is not None: + envelopes.set_subparam(value=probe_value) + probe_value = envelopes.pack() + self.lightfuzz.debug(f"probe_value_incoming (after unpacking): {probe_value}") if not isinstance(probe_value, str): - probe_value = str(probe_value) + raise ValueError( + f"probe_value_incoming should always be a string (got {type(probe_value)} / {probe_value})" + ) return probe_value def probe_value_outgoing(self, outgoing_probe_value): - self.lightfuzz.debug(f"probe_value_outgoing (before modification): {outgoing_probe_value}") - envelopes_instance = getattr(self.event, "envelopes", None) - outgoing_probe_value = envelopes_instance.add_envelopes(outgoing_probe_value) - self.lightfuzz.debug(f"probe_value_outgoing (after modification): {outgoing_probe_value}") + self.lightfuzz.debug(f"probe_value_outgoing (before packing): {outgoing_probe_value} / {self.event}") + envelopes = getattr(self.event, "envelopes", None) + if envelopes is not None: + envelopes.set_subparam(value=outgoing_probe_value) + outgoing_probe_value = envelopes.pack() + self.lightfuzz.debug(f"probe_value_outgoing (after packing): {outgoing_probe_value} / {self.event}") return outgoing_probe_value diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index b9dbb2764..450697a8d 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -5,9 +5,7 @@ class CmdILightfuzz(BaseLightfuzz): - async def fuzz(self): - cookies = self.event.data.get("assigned_cookies", {}) probe_value = self.probe_value_incoming() @@ -31,7 +29,6 @@ async def fuzz(self): echo_probe = urllib.parse.quote(echo_probe.encode(), safe="") cmdi_probe = await self.compare_probe(http_compare, self.event.data["type"], echo_probe, cookies) if cmdi_probe[3]: - if canary in cmdi_probe[3].text and "echo" not in cmdi_probe[3].text: self.lightfuzz.debug(f"canary [{canary}] found in response when sending probe [{p}]") if p == "AAAA": diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 5635774bf..b2d44e5aa 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -6,7 +6,6 @@ class CryptoLightfuzz(BaseLightfuzz): - @staticmethod def is_hex(s): try: @@ -21,7 +20,6 @@ def is_base64(s): if base64.b64encode(base64.b64decode(s)).decode() == s: return True except Exception: - return False return False @@ -75,7 +73,6 @@ def format_agnostic_encode(data, encoding, urlencode=False): @staticmethod def modify_string(input_string, action="truncate", position=None, extension_length=1): - if not isinstance(input_string, str): input_string = str(input_string) @@ -136,7 +133,7 @@ async def padding_oracle_execute(self, original_data, encoding, block_size, cook paddingblock = b"\x00" * block_size datablock = original_data[-block_size:] if possible_first_byte: - baseline_byte = b"\xFF" + baseline_byte = b"\xff" starting_pos = 0 else: baseline_byte = b"\x00" @@ -148,7 +145,6 @@ async def padding_oracle_execute(self, original_data, encoding, block_size, cook ) differ_count = 0 for i in range(starting_pos, starting_pos + 254): - byte = bytes([i]) oracle_probe = await self.compare_probe( baseline, @@ -176,7 +172,6 @@ async def padding_oracle(self, probe_value, cookies): possible_block_sizes = self.possible_block_sizes(len(data)) for block_size in possible_block_sizes: - padding_oracle_result = await self.padding_oracle_execute(data, encoding, block_size, cookies) if padding_oracle_result == None: self.lightfuzz.debug( @@ -198,7 +193,6 @@ async def padding_oracle(self, probe_value, cookies): ) async def error_string_search(self, text_dict, baseline_text): - matching_techniques = set() matching_strings = set() @@ -311,7 +305,6 @@ async def fuzz(self): if confirmed_techniques or ( "padding" in truncate_probe[3].text.lower() or "padding" in mutate_probe[3].text.lower() ): - # Padding Oracle Test if possible_block_cipher: diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index c4043e108..9ac9fa5c8 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -6,7 +6,6 @@ class PathTraversalLightfuzz(BaseLightfuzz): - async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) probe_value = self.probe_value_incoming(populate_empty=False) diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 62853964f..68fc9a118 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -38,7 +38,6 @@ def evaluate_delay(self, mean_baseline, measured_delay): return False async def fuzz(self): - cookies = self.event.data.get("assigned_cookies", {}) probe_value = self.probe_value_incoming(populate_empty=True) http_compare = self.compare_baseline( diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index 771e4b4f7..3b3c488d1 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -207,7 +207,14 @@ async def get_asn_bgpview(self, ip): return False asns_tried.add(asn) asns.append( - {"asn": asn, "subnet": subnet, "name": name, "description": description, "country": country, "emails": emails} + { + "asn": asn, + "subnet": subnet, + "name": name, + "description": description, + "country": country, + "emails": emails, + } ) if not asns: self.debug(f'No results for "{ip}"') diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 8b3cb988e..88219000e 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -178,7 +178,9 @@ def preset_from_args(self): def create_parser(self, *args, **kwargs): kwargs.update( { - "description": "Bighuge BLS OSINT Tool", "formatter_class": argparse.RawTextHelpFormatter, "epilog": self.epilog + "description": "Bighuge BLS OSINT Tool", + "formatter_class": argparse.RawTextHelpFormatter, + "epilog": self.epilog, } ) p = argparse.ArgumentParser(*args, **kwargs) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9e67f2c80..b275cc1f7 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -967,7 +967,7 @@ def presets_table(self, include_modules=True): header = ["Preset", "Category", "Description", "# Modules"] if include_modules: header.append("Modules") - for (loaded_preset, category, preset_path, original_file) in self.all_presets.values(): + for loaded_preset, category, preset_path, original_file in self.all_presets.values(): loaded_preset = loaded_preset.bake() num_modules = f"{len(loaded_preset.scan_modules):,}" row = [loaded_preset.name, category, loaded_preset.description, num_modules] diff --git a/bbot/test/test_step_1/test__module__tests.py b/bbot/test/test_step_1/test__module__tests.py index 791e58f58..e50f67a91 100644 --- a/bbot/test/test_step_1/test__module__tests.py +++ b/bbot/test/test_step_1/test__module__tests.py @@ -15,7 +15,6 @@ def test__module__tests(): - preset = Preset() # make sure each module has a .py file diff --git a/bbot/test/test_step_1/test_bbot_fastapi.py b/bbot/test/test_step_1/test_bbot_fastapi.py index add7ad099..1136963a3 100644 --- a/bbot/test/test_step_1/test_bbot_fastapi.py +++ b/bbot/test/test_step_1/test_bbot_fastapi.py @@ -17,7 +17,6 @@ def run_bbot_multiprocess(queue): def test_bbot_multiprocess(bbot_httpserver): - bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com") queue = multiprocessing.Queue() @@ -32,12 +31,10 @@ def test_bbot_multiprocess(bbot_httpserver): def test_bbot_fastapi(bbot_httpserver): - bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com") fastapi_process = start_fastapi_server() try: - # wait for the server to start with a timeout of 60 seconds start_time = time.time() while True: diff --git a/bbot/test/test_step_1/test_bloom_filter.py b/bbot/test/test_step_1/test_bloom_filter.py index 22ec4db32..f954bfbc6 100644 --- a/bbot/test/test_step_1/test_bloom_filter.py +++ b/bbot/test/test_step_1/test_bloom_filter.py @@ -6,7 +6,6 @@ @pytest.mark.asyncio async def test_bloom_filter(): - def generate_random_strings(n, length=10): """Generate a list of n random strings.""" return ["".join(random.choices(string.ascii_letters + string.digits, k=length)) for _ in range(n)] diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index dbbfe68d6..c032b44e4 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -185,7 +185,6 @@ async def test_dns_resolution(bbot_scanner): @pytest.mark.asyncio async def test_wildcards(bbot_scanner): - scan = bbot_scanner("1.1.1.1") helpers = scan.helpers @@ -634,7 +633,6 @@ def custom_lookup(query, rdtype): @pytest.mark.asyncio async def test_wildcard_deduplication(bbot_scanner): - custom_lookup = """ def custom_lookup(query, rdtype): if rdtype == "TXT" and query.strip(".").endswith("evilcorp.com"): @@ -670,7 +668,6 @@ async def handle_event(self, event): @pytest.mark.asyncio async def test_dns_raw_records(bbot_scanner): - from bbot.modules.base import BaseModule class DummyModule(BaseModule): diff --git a/bbot/test/test_step_1/test_engine.py b/bbot/test/test_step_1/test_engine.py index dbb21246f..653c3dcd6 100644 --- a/bbot/test/test_step_1/test_engine.py +++ b/bbot/test/test_step_1/test_engine.py @@ -14,7 +14,6 @@ async def test_engine(): return_errored = False class TestEngineServer(EngineServer): - CMDS = { 0: "return_thing", 1: "yield_stuff", @@ -54,7 +53,6 @@ async def yield_stuff(self, n): raise class TestEngineClient(EngineClient): - SERVER_CLASS = TestEngineServer async def return_thing(self, n): diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 39be4d704..195f08ea8 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -9,7 +9,6 @@ @pytest.mark.asyncio async def test_events(events, helpers): - scan = Scanner() await scan._prep() @@ -617,7 +616,6 @@ async def test_events(events, helpers): @pytest.mark.asyncio async def test_event_discovery_context(): - from bbot.modules.base import BaseModule scan = Scanner("evilcorp.com") diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 16b0dc9ec..2eb67cd13 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -857,7 +857,6 @@ def test_liststring_invalidfnchars(helpers): # test parameter validation @pytest.mark.asyncio async def test_parameter_validation(helpers): - getparam_valid_params = { "name", "age", diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 73fdcf23a..5b1564f12 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -16,7 +16,7 @@ def test_preset_descriptions(): # ensure very preset has a description preset = Preset() - for (loaded_preset, category, preset_path, original_filename) in preset.all_presets.values(): + for loaded_preset, category, preset_path, original_filename in preset.all_presets.values(): assert ( loaded_preset.description ), f'Preset "{loaded_preset.name}" at {original_filename} does not have a description.' @@ -68,7 +68,6 @@ def test_core(): def test_preset_yaml(clean_default_config): - import yaml preset1 = Preset( @@ -171,7 +170,6 @@ def test_preset_cache(): def test_preset_scope(): - # test target merging scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]})) assert {str(h) for h in scan.preset.target.seeds.hosts} == {"1.2.3.4/32", "evilcorp.com"} @@ -378,7 +376,6 @@ def test_preset_scope(): @pytest.mark.asyncio async def test_preset_logging(): - scan = Scanner() # test individual verbosity levels @@ -711,7 +708,6 @@ class TestModule5(BaseModule): def test_preset_include(): - # test recursive preset inclusion custom_preset_dir_1 = bbot_test_dir / "custom_preset_dir" @@ -883,7 +879,6 @@ def test_preset_module_disablement(clean_default_config): def test_preset_require_exclude(): - def get_module_flags(p): for m in p.scan_modules: preloaded = p.preloaded_module(m) diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 3c9a9832b..8f2a6bf91 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -337,7 +337,6 @@ async def test_target(bbot_scanner): @pytest.mark.asyncio async def test_blacklist_regex(bbot_scanner, bbot_httpserver): - from bbot.scanner.target import ScanBlacklist blacklist = ScanBlacklist("evilcorp.com") diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index 372f1eee6..4d324abce 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -6,7 +6,6 @@ @pytest.mark.asyncio async def test_web_engine(bbot_scanner, bbot_httpserver, httpx_mock): - from werkzeug.wrappers import Response def server_handler(request): @@ -134,7 +133,6 @@ def server_handler(request): @pytest.mark.asyncio async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): - # json conversion scan = bbot_scanner("evilcorp.com") url = "http://www.evilcorp.com/json_test?a=b" diff --git a/bbot/test/test_step_1/test_web_envelopes.py b/bbot/test/test_step_1/test_web_envelopes.py new file mode 100644 index 000000000..a5904db14 --- /dev/null +++ b/bbot/test/test_step_1/test_web_envelopes.py @@ -0,0 +1,339 @@ +import pytest + + +async def test_web_envelopes(): + from bbot.core.helpers.web.envelopes import ( + BaseEnvelope, + TextEnvelope, + HexEnvelope, + B64Envelope, + JSONEnvelope, + XMLEnvelope, + URLEnvelope, + ) + + # simple text + text_envelope = BaseEnvelope.detect("foo") + assert isinstance(text_envelope, TextEnvelope) + assert text_envelope.unpacked_data() == "foo" + assert text_envelope.subparams == {"__default__": "foo"} + expected_subparams = [([], "foo")] + assert list(text_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert text_envelope.get_subparam(subparam) == value + assert text_envelope.pack() == "foo" + assert text_envelope.num_envelopes == 0 + assert text_envelope.get_subparam() == "foo" + text_envelope.set_subparam(value="bar") + assert text_envelope.get_subparam() == "bar" + assert text_envelope.unpacked_data() == "bar" + + # simple binary + # binary_envelope = BaseEnvelope.detect("foo\x00") + # assert isinstance(binary_envelope, BinaryEnvelope) + # assert binary_envelope.unpacked_data == "foo\x00" + # assert binary_envelope.packed_data == "foo\x00" + # assert binary_envelope.subparams == {"__default__": "foo\x00"} + + # text encoded as hex + hex_envelope = BaseEnvelope.detect("706172616d") + assert isinstance(hex_envelope, HexEnvelope) + assert hex_envelope.unpacked_data(recursive=True) == "param" + hex_inner_envelope = hex_envelope.unpacked_data(recursive=False) + assert isinstance(hex_inner_envelope, TextEnvelope) + assert hex_inner_envelope.unpacked_data(recursive=False) == "param" + assert hex_inner_envelope.unpacked_data(recursive=True) == "param" + assert list(hex_envelope.get_subparams(recursive=False)) == [([], hex_inner_envelope)] + assert list(hex_envelope.get_subparams(recursive=True)) == [([], "param")] + assert hex_inner_envelope.unpacked_data() == "param" + assert hex_inner_envelope.subparams == {"__default__": "param"} + expected_subparams = [([], "param")] + assert list(hex_inner_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert hex_inner_envelope.get_subparam(subparam) == value + assert hex_envelope.pack() == "706172616d" + assert hex_envelope.num_envelopes == 1 + assert hex_envelope.get_subparam() == "param" + hex_envelope.set_subparam(value="asdf") + assert hex_envelope.get_subparam() == "asdf" + assert hex_envelope.unpacked_data() == "asdf" + assert hex_envelope.pack() == "61736466" + + # text encoded as base64 + base64_envelope = BaseEnvelope.detect("cGFyYW0=") + assert isinstance(base64_envelope, B64Envelope) + assert base64_envelope.unpacked_data() == "param" + base64_inner_envelope = base64_envelope.unpacked_data(recursive=False) + assert isinstance(base64_inner_envelope, TextEnvelope) + assert list(base64_envelope.get_subparams(recursive=False)) == [([], base64_inner_envelope)] + assert list(base64_envelope.get_subparams()) == [([], "param")] + assert base64_inner_envelope.pack() == "param" + assert base64_inner_envelope.unpacked_data() == "param" + assert base64_inner_envelope.subparams == {"__default__": "param"} + expected_subparams = [([], "param")] + assert list(base64_inner_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert base64_inner_envelope.get_subparam(subparam) == value + assert base64_envelope.num_envelopes == 1 + base64_envelope.set_subparam(value="asdf") + assert base64_envelope.get_subparam() == "asdf" + assert base64_envelope.unpacked_data() == "asdf" + assert base64_envelope.pack() == "YXNkZg==" + + # test inside hex inside base64 + hex_envelope = BaseEnvelope.detect("634746795957303d") + assert isinstance(hex_envelope, HexEnvelope) + assert hex_envelope.get_subparam() == "param" + assert hex_envelope.unpacked_data() == "param" + base64_envelope = hex_envelope.unpacked_data(recursive=False) + assert isinstance(base64_envelope, B64Envelope) + assert base64_envelope.get_subparam() == "param" + assert base64_envelope.unpacked_data() == "param" + text_envelope = base64_envelope.unpacked_data(recursive=False) + assert isinstance(text_envelope, TextEnvelope) + assert text_envelope.get_subparam() == "param" + assert text_envelope.unpacked_data() == "param" + hex_envelope.set_subparam(value="asdf") + assert hex_envelope.get_subparam() == "asdf" + assert hex_envelope.unpacked_data() == "asdf" + assert text_envelope.get_subparam() == "asdf" + assert text_envelope.unpacked_data() == "asdf" + assert base64_envelope.get_subparam() == "asdf" + assert base64_envelope.unpacked_data() == "asdf" + + # URL-encoded text + url_encoded_envelope = BaseEnvelope.detect("a%20b%20c") + assert isinstance(url_encoded_envelope, URLEnvelope) + assert url_encoded_envelope.pack() == "a%20b%20c" + assert url_encoded_envelope.unpacked_data() == "a b c" + url_inner_envelope = url_encoded_envelope.unpacked_data(recursive=False) + assert isinstance(url_inner_envelope, TextEnvelope) + assert url_inner_envelope.unpacked_data(recursive=False) == "a b c" + assert url_inner_envelope.unpacked_data(recursive=True) == "a b c" + assert list(url_encoded_envelope.get_subparams(recursive=False)) == [([], url_inner_envelope)] + assert list(url_encoded_envelope.get_subparams(recursive=True)) == [([], "a b c")] + assert url_inner_envelope.pack() == "a b c" + assert url_inner_envelope.unpacked_data() == "a b c" + assert url_inner_envelope.subparams == {"__default__": "a b c"} + expected_subparams = [([], "a b c")] + assert list(url_inner_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert url_inner_envelope.get_subparam(subparam) == value + assert url_encoded_envelope.num_envelopes == 1 + url_encoded_envelope.set_subparam(value="a s d f") + assert url_encoded_envelope.get_subparam() == "a s d f" + assert url_encoded_envelope.unpacked_data() == "a s d f" + assert url_encoded_envelope.pack() == "a%20s%20d%20f" + + # json + json_envelope = BaseEnvelope.detect('{"param1": "val1", "param2": {"param3": "val3"}}') + assert isinstance(json_envelope, JSONEnvelope) + assert json_envelope.pack() == '{"param1": "val1", "param2": {"param3": "val3"}}' + assert json_envelope.unpacked_data() == {"param1": "val1", "param2": {"param3": "val3"}} + assert json_envelope.unpacked_data(recursive=False) == {"param1": "val1", "param2": {"param3": "val3"}} + assert json_envelope.unpacked_data(recursive=True) == {"param1": "val1", "param2": {"param3": "val3"}} + assert json_envelope.subparams == {"param1": "val1", "param2": {"param3": "val3"}} + expected_subparams = [ + (["param1"], "val1"), + (["param2", "param3"], "val3"), + ] + assert list(json_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert json_envelope.get_subparam(subparam) == value + json_envelope.selected_subparam = ["param2", "param3"] + assert json_envelope.get_subparam() == "val3" + assert json_envelope.num_envelopes == 1 + + # xml + xml_envelope = BaseEnvelope.detect( + 'val1val3' + ) + assert isinstance(xml_envelope, XMLEnvelope) + assert ( + xml_envelope.pack() + == '\nval1val3' + ) + assert xml_envelope.unpacked_data() == { + "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + assert xml_envelope.unpacked_data(recursive=False) == { + "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + assert xml_envelope.unpacked_data(recursive=True) == { + "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + assert xml_envelope.subparams == { + "root": {"param1": {"@attr": "attr1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + expected_subparams = [ + (["root", "param1", "@attr"], "attr1"), + (["root", "param1", "#text"], "val1"), + (["root", "param2", "param3"], "val3"), + ] + assert list(xml_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert xml_envelope.get_subparam(subparam) == value + assert xml_envelope.num_envelopes == 1 + + # json inside base64 + base64_json_envelope = BaseEnvelope.detect("eyJwYXJhbTEiOiAidmFsMSIsICJwYXJhbTIiOiB7InBhcmFtMyI6ICJ2YWwzIn19") + assert isinstance(base64_json_envelope, B64Envelope) + assert base64_json_envelope.pack() == "eyJwYXJhbTEiOiAidmFsMSIsICJwYXJhbTIiOiB7InBhcmFtMyI6ICJ2YWwzIn19" + assert base64_json_envelope.unpacked_data() == {"param1": "val1", "param2": {"param3": "val3"}} + base64_inner_envelope = base64_json_envelope.unpacked_data(recursive=False) + assert isinstance(base64_inner_envelope, JSONEnvelope) + assert base64_inner_envelope.pack() == '{"param1": "val1", "param2": {"param3": "val3"}}' + assert base64_inner_envelope.unpacked_data() == {"param1": "val1", "param2": {"param3": "val3"}} + assert base64_inner_envelope.subparams == {"param1": "val1", "param2": {"param3": "val3"}} + expected_subparams = [ + (["param1"], "val1"), + (["param2", "param3"], "val3"), + ] + assert list(base64_json_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert base64_json_envelope.get_subparam(subparam) == value + assert base64_json_envelope.num_envelopes == 2 + with pytest.raises(ValueError): + assert base64_json_envelope.get_subparam() + base64_json_envelope.selected_subparam = ["param2", "param3"] + assert base64_json_envelope.get_subparam() == "val3" + + # xml inside url inside hex inside base64 + nested_xml_envelope = BaseEnvelope.detect( + "MjUzMzYzMjUzNzMyMjUzNjY2MjUzNjY2MjUzNzM0MjUzMzY1MjUzMzYzMjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMxMjUzMjMwMjUzNjMxMjUzNzM0MjUzNzM0MjUzNzMyMjUzMzY0MjUzMjMyMjUzNzM2MjUzNjMxMjUzNjYzMjUzMzMxMjUzMjMyMjUzMzY1MjUzNzM2MjUzNjMxMjUzNjYzMjUzMzMxMjUzMzYzMjUzMjY2MjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMxMjUzMzY1MjUzMzYzMjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMyMjUzMzY1MjUzMzYzMjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMzMjUzMzY1MjUzNzM2MjUzNjMxMjUzNjYzMjUzMzMzMjUzMzYzMjUzMjY2MjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMzMjUzMzY1MjUzMzYzMjUzMjY2MjUzNzMwMjUzNjMxMjUzNzMyMjUzNjMxMjUzNjY0MjUzMzMyMjUzMzY1MjUzMzYzMjUzMjY2MjUzNzMyMjUzNjY2MjUzNjY2MjUzNzM0MjUzMzY1" + ) + assert isinstance(nested_xml_envelope, B64Envelope) + assert nested_xml_envelope.unpacked_data() == { + "root": {"param1": {"@attr": "val1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + assert ( + nested_xml_envelope.pack() + == "MjUzMzQzMjUzMzQ2Nzg2ZDZjMjUzMjMwNzY2NTcyNzM2OTZmNmUyNTMzNDQyNTMyMzIzMTJlMzAyNTMyMzIyNTMyMzA2NTZlNjM2ZjY0Njk2ZTY3MjUzMzQ0MjUzMjMyNzU3NDY2MmQzODI1MzIzMjI1MzM0NjI1MzM0NTI1MzA0MTI1MzM0MzcyNmY2Zjc0MjUzMzQ1MjUzMzQzNzA2MTcyNjE2ZDMxMjUzMjMwNjE3NDc0NzIyNTMzNDQyNTMyMzI3NjYxNmMzMTI1MzIzMjI1MzM0NTc2NjE2YzMxMjUzMzQzMmY3MDYxNzI2MTZkMzEyNTMzNDUyNTMzNDM3MDYxNzI2MTZkMzIyNTMzNDUyNTMzNDM3MDYxNzI2MTZkMzMyNTMzNDU3NjYxNmMzMzI1MzM0MzJmNzA2MTcyNjE2ZDMzMjUzMzQ1MjUzMzQzMmY3MDYxNzI2MTZkMzIyNTMzNDUyNTMzNDMyZjcyNmY2Zjc0MjUzMzQ1" + ) + inner_hex_envelope = nested_xml_envelope.unpacked_data(recursive=False) + assert isinstance(inner_hex_envelope, HexEnvelope) + assert ( + inner_hex_envelope.pack() + == "253343253346786d6c25323076657273696f6e253344253232312e30253232253230656e636f64696e672533442532327574662d38253232253346253345253041253343726f6f74253345253343706172616d312532306174747225334425323276616c3125323225334576616c312533432f706172616d31253345253343706172616d32253345253343706172616d3325334576616c332533432f706172616d332533452533432f706172616d322533452533432f726f6f74253345" + ) + inner_url_envelope = inner_hex_envelope.unpacked_data(recursive=False) + assert isinstance(inner_url_envelope, URLEnvelope) + assert ( + inner_url_envelope.pack() + == r"%3C%3Fxml%20version%3D%221.0%22%20encoding%3D%22utf-8%22%3F%3E%0A%3Croot%3E%3Cparam1%20attr%3D%22val1%22%3Eval1%3C/param1%3E%3Cparam2%3E%3Cparam3%3Eval3%3C/param3%3E%3C/param2%3E%3C/root%3E" + ) + inner_xml_envelope = inner_url_envelope.unpacked_data(recursive=False) + assert isinstance(inner_xml_envelope, XMLEnvelope) + assert ( + inner_xml_envelope.pack() + == '\nval1val3' + ) + assert inner_xml_envelope.unpacked_data() == { + "root": {"param1": {"@attr": "val1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + assert inner_xml_envelope.subparams == { + "root": {"param1": {"@attr": "val1", "#text": "val1"}, "param2": {"param3": "val3"}} + } + expected_subparams = [ + (["root", "param1", "@attr"], "val1"), + (["root", "param1", "#text"], "val1"), + (["root", "param2", "param3"], "val3"), + ] + assert list(nested_xml_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert nested_xml_envelope.get_subparam(subparam) == value + assert nested_xml_envelope.num_envelopes == 4 + + # manipulating text inside hex + hex_envelope = BaseEnvelope.detect("706172616d") + expected_subparams = [([], "param")] + assert list(hex_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert hex_envelope.get_subparam(subparam) == value + hex_envelope.set_subparam([], "asdf") + expected_subparams = [([], "asdf")] + assert list(hex_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert hex_envelope.get_subparam(subparam) == value + assert hex_envelope.unpacked_data() == "asdf" + + # manipulating json inside base64 + base64_json_envelope = BaseEnvelope.detect("eyJwYXJhbTEiOiAidmFsMSIsICJwYXJhbTIiOiB7InBhcmFtMyI6ICJ2YWwzIn19") + expected_subparams = [ + (["param1"], "val1"), + (["param2", "param3"], "val3"), + ] + assert list(base64_json_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert base64_json_envelope.get_subparam(subparam) == value + base64_json_envelope.set_subparam(["param1"], {"asdf": [None], "fdsa": 1.0}) + expected_subparams = [ + (["param1", "asdf"], [None]), + (["param1", "fdsa"], 1.0), + (["param2", "param3"], "val3"), + ] + assert list(base64_json_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert base64_json_envelope.get_subparam(subparam) == value + base64_json_envelope.set_subparam(["param2", "param3"], {"1234": [None], "4321": 1.0}) + expected_subparams = [ + (["param1", "asdf"], [None]), + (["param1", "fdsa"], 1.0), + (["param2", "param3", "1234"], [None]), + (["param2", "param3", "4321"], 1.0), + ] + assert list(base64_json_envelope.get_subparams()) == expected_subparams + base64_json_envelope.set_subparam(["param2"], None) + expected_subparams = [ + (["param1", "asdf"], [None]), + (["param1", "fdsa"], 1.0), + (["param2"], None), + ] + assert list(base64_json_envelope.get_subparams()) == expected_subparams + + # xml inside url inside base64 + xml_envelope = BaseEnvelope.detect( + "JTNDP3htbCUyMHZlcnNpb249JTIyMS4wJTIyJTIwZW5jb2Rpbmc9JTIydXRmLTglMjI/JTNFJTBBJTNDcm9vdCUzRSUzQ3BhcmFtMSUyMGF0dHI9JTIydmFsMSUyMiUzRXZhbDElM0MvcGFyYW0xJTNFJTNDcGFyYW0yJTNFJTNDcGFyYW0zJTNFdmFsMyUzQy9wYXJhbTMlM0UlM0MvcGFyYW0yJTNFJTNDL3Jvb3QlM0U=" + ) + assert ( + xml_envelope.pack() + == "JTNDJTNGeG1sJTIwdmVyc2lvbiUzRCUyMjEuMCUyMiUyMGVuY29kaW5nJTNEJTIydXRmLTglMjIlM0YlM0UlMEElM0Nyb290JTNFJTNDcGFyYW0xJTIwYXR0ciUzRCUyMnZhbDElMjIlM0V2YWwxJTNDL3BhcmFtMSUzRSUzQ3BhcmFtMiUzRSUzQ3BhcmFtMyUzRXZhbDMlM0MvcGFyYW0zJTNFJTNDL3BhcmFtMiUzRSUzQy9yb290JTNF" + ) + expected_subparams = [ + (["root", "param1", "@attr"], "val1"), + (["root", "param1", "#text"], "val1"), + (["root", "param2", "param3"], "val3"), + ] + assert list(xml_envelope.get_subparams()) == expected_subparams + xml_envelope.set_subparam(["root", "param1", "@attr"], "asdf") + expected_subparams = [ + (["root", "param1", "@attr"], "asdf"), + (["root", "param1", "#text"], "val1"), + (["root", "param2", "param3"], "val3"), + ] + assert list(xml_envelope.get_subparams()) == expected_subparams + assert ( + xml_envelope.pack() + == "JTNDJTNGeG1sJTIwdmVyc2lvbiUzRCUyMjEuMCUyMiUyMGVuY29kaW5nJTNEJTIydXRmLTglMjIlM0YlM0UlMEElM0Nyb290JTNFJTNDcGFyYW0xJTIwYXR0ciUzRCUyMmFzZGYlMjIlM0V2YWwxJTNDL3BhcmFtMSUzRSUzQ3BhcmFtMiUzRSUzQ3BhcmFtMyUzRXZhbDMlM0MvcGFyYW0zJTNFJTNDL3BhcmFtMiUzRSUzQy9yb290JTNF" + ) + xml_envelope.set_subparam(["root", "param2", "param3"], {"1234": [None], "4321": 1.0}) + expected_subparams = [ + (["root", "param1", "@attr"], "asdf"), + (["root", "param1", "#text"], "val1"), + (["root", "param2", "param3", "1234"], [None]), + (["root", "param2", "param3", "4321"], 1.0), + ] + assert list(xml_envelope.get_subparams()) == expected_subparams + + # null + null_envelope = BaseEnvelope.detect("null") + assert isinstance(null_envelope, JSONEnvelope) + assert null_envelope.unpacked_data() == None + assert null_envelope.pack() == "null" + expected_subparams = [([], None)] + assert list(null_envelope.get_subparams()) == expected_subparams + for subparam, value in expected_subparams: + assert null_envelope.get_subparam(subparam) == value + + tiny_base64 = BaseEnvelope.detect("YWJi") + assert isinstance(tiny_base64, TextEnvelope) diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py index 77a86153c..b2b49717c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py @@ -55,8 +55,8 @@ def set_target(self, target): def check(self, module_test, events): assert any( e.type == "FINDING" - and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" - in e.data["description"] - for e in events + and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" + in e.data["description"] + for e in events ), "Failed to emit FINDING" assert any("baddns-cname" in e.tags for e in events), "Failed to add baddns tag" diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 915ca7d58..b0c6b723a 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -499,9 +499,7 @@ def check(self, module_test, events): found_htmltags_img = False for e in events: - if e.type == "WEB_PARAMETER": - if e.data["description"] == "HTTP Extracted Parameter [jqueryget] (GET jquery Submodule)": found_jquery_get = True if e.data["original_value"] == "value1": @@ -552,7 +550,6 @@ def check(self, module_test, events): class TestExcavateParameterExtraction_postformnoaction(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER @@ -574,7 +571,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(respond_args=respond_args) def check(self, module_test, events): - excavate_getparam_extraction = False for e in events: if e.type == "WEB_PARAMETER": @@ -606,7 +602,6 @@ def check(self, module_test, events): class TestExcavateParameterExtraction_relativeurl(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER @@ -631,7 +626,6 @@ class TestExcavateParameterExtraction_relativeurl(ModuleTestBase): root_page_html = "Root page" async def setup_after_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data(self.primary_page_html) module_test.httpserver.expect_request("/secondary").respond_with_data(self.secondary_page_html) module_test.httpserver.expect_request("/root.html").respond_with_data(self.root_page_html) @@ -731,7 +725,6 @@ def check(self, module_test, events): class TestExcavateParameterExtraction_inputtagnovalue(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER @@ -1212,7 +1205,6 @@ class TestExcavate(ModuleTestBase): config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} async def setup_before_prep(self, module_test): - response_data = """ ftp://ftp.test.notreal \\nhttps://www1.test.notreal @@ -1300,13 +1292,11 @@ def check(self, module_test, events): class TestExcavateHeaders_blacklist(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] modules_overrides = ["excavate", "httpx", "hunt"] config_overrides = {"web": {"spider_distance": 1, "spider_depth": 1}} async def setup_before_prep(self, module_test): - module_test.httpserver.expect_request("/").respond_with_data( "

test

", status=200, @@ -1320,7 +1310,6 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - found_first_cookie = False found_second_cookie = False found_third_cookie = False diff --git a/bbot/test/test_step_2/module_tests/test_module_gowitness.py b/bbot/test/test_step_2/module_tests/test_module_gowitness.py index 2d6dc2cd8..6090fbb1d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_gowitness.py +++ b/bbot/test/test_step_2/module_tests/test_module_gowitness.py @@ -101,6 +101,4 @@ class TestGoWitnessWithBlob(TestGowitness): def check(self, module_test, events): webscreenshots = [e for e in events if e.type == "WEBSCREENSHOT"] assert webscreenshots, "failed to raise WEBSCREENSHOT events" - assert all( - "blob" in e.data and e.data["blob"] for e in webscreenshots - ), "blob not found in WEBSCREENSHOT data" + assert all("blob" in e.data and e.data["blob"] for e in webscreenshots), "blob not found in WEBSCREENSHOT data" diff --git a/bbot/test/test_step_2/module_tests/test_module_hunt.py b/bbot/test/test_step_2/module_tests/test_module_hunt.py index 0ce8e9353..867a2565c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_hunt.py +++ b/bbot/test/test_step_2/module_tests/test_module_hunt.py @@ -23,7 +23,6 @@ def check(self, module_test, events): class TestHunt_Multiple(TestHunt): - async def setup_after_prep(self, module_test): expect_args = {"method": "GET", "uri": "/"} respond_args = {"response_data": 'ping'} diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 72d607c0c..b54a1d277 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -34,7 +34,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def request_handler(self, request): - qs = str(request.query_string.decode()) if "filename=" in qs: value = qs.split("=")[1] @@ -52,11 +51,9 @@ def request_handler(self, request): return Response("file not found", status=500) def check(self, module_test, events): - web_parameter_emitted = False pathtraversal_finding_emitted = False for e in events: - if e.type == "WEB_PARAMETER": if "HTTP Extracted Parameter [filename]" in e.data["description"]: web_parameter_emitted = True @@ -74,7 +71,6 @@ def check(self, module_test, events): # Path Traversal Absolute path class Test_Lightfuzz_path_absolute(Test_Lightfuzz_path_singledot): - etc_passwd = """ root:x:0:0:root:/root:/bin/bash daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin @@ -87,7 +83,6 @@ class Test_Lightfuzz_path_absolute(Test_Lightfuzz_path_singledot): """ async def setup_after_prep(self, module_test): - expect_args = {"method": "GET", "uri": "/images", "query_string": "filename=/etc/passwd"} respond_args = {"response_data": self.etc_passwd} module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) @@ -104,7 +99,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args) def check(self, module_test, events): - web_parameter_emitted = False pathtraversal_finding_emitted = False for e in events: @@ -156,7 +150,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def check(self, module_test, events): - web_parameter_emitted = False ssti_finding_emitted = False for e in events: @@ -189,7 +182,6 @@ class Test_Lightfuzz_xss(ModuleTestBase): } def request_handler(self, request): - qs = str(request.query_string.decode()) parameter_block = """ @@ -219,7 +211,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def check(self, module_test, events): - web_parameter_emitted = False xss_finding_emitted = False for e in events: @@ -238,12 +229,8 @@ def check(self, module_test, events): # Base64 Envelope XSS Detection class Test_Lightfuzz_envelope_base64(Test_Lightfuzz_xss): def request_handler(self, request): - qs = str(request.query_string.decode()) - print("****") - print(qs) - parameter_block = """
@@ -266,7 +253,6 @@ def request_handler(self, request): return Response(parameter_block, status=200) def check(self, module_test, events): - web_parameter_emitted = False xss_finding_emitted = False for e in events: @@ -307,7 +293,6 @@ def request_handler(self, request): try: # Decode the hex value decoded_value = bytes.fromhex(unquote(value)).decode() - print(f"Decoded hex value: {decoded_value}") # Parse the decoded value as JSON json_data = json.loads(decoded_value) @@ -337,9 +322,6 @@ class Test_Lightfuzz_envelope_jsonb64(Test_Lightfuzz_envelope_base64): def request_handler(self, request): qs = str(request.query_string.decode()) - print("****") - print(qs) - parameter_block = """
@@ -357,7 +339,6 @@ def request_handler(self, request): try: # Base64 decode the value decoded_value = base64.b64decode(unquote(value)).decode() - print(f"Decoded base64 value: {decoded_value}") # Parse the decoded value as JSON json_data = json.loads(decoded_value) @@ -387,9 +368,6 @@ class Test_Lightfuzz_envelope_xmlb64(Test_Lightfuzz_envelope_base64): def request_handler(self, request): qs = str(request.query_string.decode()) - print("****") - print(qs) - parameter_block = """
@@ -407,7 +385,6 @@ def request_handler(self, request): try: # Base64 decode the value decoded_value = base64.b64decode(unquote(value)).decode() - print(f"Decoded base64 value: {decoded_value}") # Parse the decoded value as XML root = ET.fromstring(decoded_value) @@ -458,7 +435,6 @@ def request_handler(self, request): return Response(parameter_block, status=200) async def setup_after_prep(self, module_test): - module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) @@ -557,8 +533,6 @@ class Test_Lightfuzz_sqli(ModuleTestBase): } def request_handler(self, request): - print("((((") - print(request) qs = str(request.query_string.decode()) parameter_block = """
@@ -594,7 +568,6 @@ def request_handler(self, request): return Response(parameter_block, status=200) async def setup_after_prep(self, module_test): - module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) @@ -631,7 +604,6 @@ class Test_Lightfuzz_sqli_post(ModuleTestBase): } def request_handler(self, request): - parameter_block = """
@@ -642,7 +614,6 @@ def request_handler(self, request): """ if "search" in request.form.keys(): - value = request.form["search"] sql_block_normal = f""" @@ -665,7 +636,6 @@ def request_handler(self, request): return Response(parameter_block, status=200) async def setup_after_prep(self, module_test): - module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) @@ -691,7 +661,6 @@ def check(self, module_test, events): # disable_post test class Test_Lightfuzz_disable_post(Test_Lightfuzz_sqli_post): - config_overrides = { "interactsh_disable": True, "modules": { @@ -723,7 +692,6 @@ def check(self, module_test, events): # SQLI Single Quote/Two Single Quote (headers) class Test_Lightfuzz_sqli_headers(Test_Lightfuzz_sqli): - async def setup_after_prep(self, module_test): module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" expect_args = re.compile("/") @@ -751,7 +719,6 @@ async def setup_after_prep(self, module_test): module_test.scan.target.seeds.events = set(seed_events) def request_handler(self, request): - placeholder_block = """

placeholder

@@ -792,9 +759,7 @@ def check(self, module_test, events): # SQLI Single Quote/Two Single Quote (cookies) class Test_Lightfuzz_sqli_cookies(Test_Lightfuzz_sqli): - async def setup_after_prep(self, module_test): - module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) @@ -821,7 +786,6 @@ async def setup_after_prep(self, module_test): module_test.scan.target.seeds.events = set(seed_events) def request_handler(self, request): - placeholder_block = """

placeholder

@@ -850,7 +814,6 @@ def request_handler(self, request): return Response(placeholder_block, status=200) def check(self, module_test, events): - sqli_finding_emitted = False for e in events: if e.type == "FINDING": @@ -864,7 +827,6 @@ def check(self, module_test, events): # SQLi Delay Probe class Test_Lightfuzz_sqli_delay(Test_Lightfuzz_sqli): - def request_handler(self, request): from time import sleep @@ -898,7 +860,6 @@ def request_handler(self, request): return Response(parameter_block, status=200) def check(self, module_test, events): - web_parameter_emitted = False sqldelay_finding_emitted = False for e in events: @@ -931,12 +892,10 @@ class Test_Lightfuzz_serial_errorresolution(ModuleTestBase): } async def setup_after_prep(self, module_test): - expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def request_handler(self, request): - dotnet_serial_error = """ Description: An unhandled exception occurred during the execution of the current web request. Please review the stack trace for more information about the error and where it originated in the code. @@ -991,13 +950,11 @@ def request_handler(self, request): if post_params["__VIEWSTATE"] != "/wEPDwULLTE5MTI4MzkxNjVkZNt7ICM+GixNryV6ucx+srzhXlwP": return Response(dotnet_serial_error, status=500) if post_params["TextBox1"] == "AAEAAAD/////AQAAAAAAAAAGAQAAAAdndXN0YXZvCw==": - return Response(dotnet_serial_error_resolved, status=200) else: return Response(dotnet_serial_error, status=500) def check(self, module_test, events): - excavate_extracted_form_parameter = False excavate_extracted_form_parameter_details = False lightfuzz_serial_detect_errorresolution = False @@ -1034,9 +991,7 @@ def check(self, module_test, events): # Serialization Module (Error Differential) class Test_Lightfuzz_serial_errordifferential(Test_Lightfuzz_serial_errorresolution): - def request_handler(self, request): - java_serial_error = """

Internal Server Error

@@ -1066,17 +1021,14 @@ def request_handler(self, request): cookies = request.cookies if "session" not in cookies.keys(): - response = Response(java_serial_html, status=200) response.set_cookie("session", value="", max_age=3600, httponly=True) return response else: if cookies["session"] == "rO0ABXQABHRlc3Q=": - return Response(java_serial_error_keyword, status=500) else: - return Response(java_serial_error, status=500) def check(self, module_test, events): @@ -1085,13 +1037,10 @@ def check(self, module_test, events): for e in events: if e.type == "WEB_PARAMETER": - if e.data["description"] == "Set-Cookie Assigned Cookie [session]" and e.data["type"] == "COOKIE": excavate_extracted_cookie_parameter = True if e.type == "FINDING": - print(e.data["description"]) - if ( e.data["description"] == "POSSIBLE Unsafe Deserialization. Parameter: [session] Parameter Type: [COOKIE] Technique: [Differential Error Analysis] Error-String: [cannot cast java.lang.string] Payload: [java_base64_string_error]" @@ -1118,7 +1067,6 @@ class Test_Lightfuzz_cmdi(ModuleTestBase): } def request_handler(self, request): - qs = str(request.query_string.decode()) parameter_block = """ @@ -1150,13 +1098,11 @@ def request_handler(self, request): return Response(parameter_block, status=200) async def setup_after_prep(self, module_test): - module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def check(self, module_test, events): - web_parameter_emitted = False cmdi_echocanary_finding_emitted = False for e in events: @@ -1177,7 +1123,6 @@ def check(self, module_test, events): # CMDi interactsh class Test_Lightfuzz_cmdi_interactsh(Test_Lightfuzz_cmdi): - @staticmethod def extract_subdomain_tag(data): pattern = r"search=.+%26%26%20nslookup%20(.+)\.fakedomain\.fakeinteractsh.com%20%26%26" @@ -1195,7 +1140,6 @@ def extract_subdomain_tag(data): } def request_handler(self, request): - qs = str(request.query_string.decode()) parameter_block = """ @@ -1208,7 +1152,6 @@ def request_handler(self, request): """ if "search=" in qs: - subdomain_tag = None subdomain_tag = self.extract_subdomain_tag(request.full_path) @@ -1224,22 +1167,18 @@ async def setup_before_prep(self, module_test): ) async def setup_after_prep(self, module_test): - expect_args = re.compile("/") module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def check(self, module_test, events): - web_parameter_emitted = False cmdi_interacttsh_finding_emitted = False for e in events: if e.type == "WEB_PARAMETER": - print(e.data["description"]) if "HTTP Extracted Parameter [search]" in e.data["description"]: web_parameter_emitted = True if e.type == "VULNERABILITY": - print(e.data["description"]) if ( "OS Command Injection (OOB Interaction) Type: [GETPARAM] Parameter Name: [search] Probe: [&&]" in e.data["description"] @@ -1250,63 +1189,62 @@ def check(self, module_test, events): assert cmdi_interacttsh_finding_emitted, "interactsh CMDi FINDING not emitted" -class Test_Lightfuzz_speculative(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] - modules_overrides = ["httpx", "excavate", "paramminer_getparams", "lightfuzz"] - config_overrides = { - "interactsh_disable": True, - "modules": { - "lightfuzz": {"enabled_submodules": ["xss"]}, - "paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}, - }, - } - - def request_handler(self, request): - - qs = str(request.query_string.decode()) - parameter_block = """ - { - "search": 1, - "common": 1 - } - """ - if "search=" in qs: - value = qs.split("=")[1] - if "&" in value: - value = value.split("&")[0] - xss_block = f""" -
-

0 search results for '{unquote(value)}'

-
-
- """ - return Response(xss_block, status=200) - return Response(parameter_block, status=200, headers={"Content-Type": "application/json"}) - - async def setup_after_prep(self, module_test): - module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" - expect_args = re.compile("/") - module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) - - def check(self, module_test, events): - excavate_json_extraction = False - xss_finding_emitted = False - - for e in events: - if e.type == "WEB_PARAMETER": - if "HTTP Extracted Parameter (speculative from json content) [search]" in e.data["description"]: - excavate_json_extraction = True - - if e.type == "FINDING": - if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags" in e.data["description"]: - xss_finding_emitted = True - - assert excavate_json_extraction, "Excavate failed to extract json parameter" - assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" +# class Test_Lightfuzz_speculative(ModuleTestBase): +# targets = ["http://127.0.0.1:8888/"] +# modules_overrides = ["httpx", "excavate", "paramminer_getparams", "lightfuzz"] +# config_overrides = { +# "interactsh_disable": True, +# "modules": { +# "lightfuzz": {"enabled_submodules": ["xss"]}, +# "paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}, +# }, +# } + +# def request_handler(self, request): + +# qs = str(request.query_string.decode()) +# parameter_block = """ +# { +# "search": 1, +# "common": 1 +# } +# """ +# if "search=" in qs: +# value = qs.split("=")[1] +# if "&" in value: +# value = value.split("&")[0] +# xss_block = f""" +#
+#

0 search results for '{unquote(value)}'

+#
+#
+# """ +# return Response(xss_block, status=200) +# return Response(parameter_block, status=200, headers={"Content-Type": "application/json"}) + +# async def setup_after_prep(self, module_test): +# module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" +# expect_args = re.compile("/") +# module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + +# def check(self, module_test, events): +# excavate_json_extraction = False +# xss_finding_emitted = False + +# for e in events: +# if e.type == "WEB_PARAMETER": +# if "HTTP Extracted Parameter (speculative from json content) [search]" in e.data["description"]: +# excavate_json_extraction = True + +# if e.type == "FINDING": +# if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags" in e.data["description"]: +# xss_finding_emitted = True + +# assert excavate_json_extraction, "Excavate failed to extract json parameter" +# assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" class Test_Lightfuzz_crypto_error(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] modules_overrides = ["httpx", "excavate", "lightfuzz"] config_overrides = { @@ -1317,7 +1255,6 @@ class Test_Lightfuzz_crypto_error(ModuleTestBase): } def request_handler(self, request): - qs = str(request.query_string.decode()) parameter_block = """ @@ -1351,7 +1288,6 @@ def check(self, module_test, events): cryptoerror_finding_emitted = False for e in events: - if e.type == "WEB_PARAMETER": if "HTTP Extracted Parameter [secret] (GET Form Submodule)" in e.data["description"]: cryptoerror_parameter_extracted = True @@ -1366,7 +1302,6 @@ def check(self, module_test, events): class Test_Lightfuzz_crypto_error_falsepositive(ModuleTestBase): - targets = ["http://127.0.0.1:8888/"] modules_overrides = ["httpx", "excavate", "lightfuzz"] config_overrides = { @@ -1410,77 +1345,80 @@ def check(self, module_test, events): ), "Crypto Error Message FINDING was emitted (it is an intentional false positive)" -class Test_PaddingOracleDetection(ModuleTestBase): - - targets = ["http://127.0.0.1:8888"] - modules_overrides = ["httpx", "excavate", "lightfuzz"] - config_overrides = { - "interactsh_disable": True, - "modules": { - "lightfuzz": { - "enabled_submodules": ["crypto"], - } - }, - } - - def request_handler(self, request): - encrypted_value = quote( - "dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==" - ) - default_html_response = f""" - - - - - - - - - """ - - if "/decrypt" in request.url and request.method == "POST": - if request.form and request.form["encrypted_data"]: - encrypted_data = request.form["encrypted_data"] - if "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm" in encrypted_data: - response_content = "Padding error detected" - elif "4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg" in encrypted_data: - response_content = "DIFFERENT CRYPTOGRAPHIC ERROR" - elif "AAAAAAA" in encrypted_data: - response_content = "YET DIFFERENT CRYPTOGRAPHIC ERROR" - else: - response_content = "Decryption failed" - - return Response(response_content, status=200) - else: - return Response(default_html_response, status=200) - - async def setup_after_prep(self, module_test): - module_test.set_expect_requests_handler(expect_args=re.compile(".*"), request_handler=self.request_handler) - - def check(self, module_test, events): - - web_parameter_extracted = False - cryptographic_parameter_finding = False - padding_oracle_detected = False - for e in events: - - if e.type == "WEB_PARAMETER": - if "HTTP Extracted Parameter [encrypted_data] (POST Form" in e.data["description"]: - web_parameter_extracted = True - if e.type == "FINDING": - if ( - e.data["description"] - == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [url-encoded]" - ): - cryptographic_parameter_finding = True - - if e.type == "VULNERABILITY": - if ( - e.data["description"] - == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [url-encoded]" - ): - padding_oracle_detected = True - - assert web_parameter_extracted, "Web parameter was not extracted" - assert cryptographic_parameter_finding, "Cryptographic parameter not detected" - assert padding_oracle_detected, "Padding oracle vulnerability was not detected" +# class Test_PaddingOracleDetection(ModuleTestBase): + +# targets = ["http://127.0.0.1:8888"] +# modules_overrides = ["httpx", "excavate", "lightfuzz"] +# config_overrides = { +# "interactsh_disable": True, +# "modules": { +# "lightfuzz": { +# "enabled_submodules": ["crypto"], +# } +# }, +# } + +# def request_handler(self, request): +# encrypted_value = quote( +# "dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==" +# ) +# default_html_response = f""" +# +# +#
+# +# +#
+# +# +# """ + +# if "/decrypt" in request.url and request.method == "POST": +# if request.form and request.form["encrypted_data"]: +# encrypted_data = request.form["encrypted_data"] +# if "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm" in encrypted_data: +# response_content = "Padding error detected" +# elif "4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg" in encrypted_data: +# response_content = "DIFFERENT CRYPTOGRAPHIC ERROR" +# elif "AAAAAAA" in encrypted_data: +# response_content = "YET DIFFERENT CRYPTOGRAPHIC ERROR" +# else: +# response_content = "Decryption failed" + +# return Response(response_content, status=200) +# else: +# return Response(default_html_response, status=200) + +# async def setup_after_prep(self, module_test): +# module_test.set_expect_requests_handler(expect_args=re.compile(".*"), request_handler=self.request_handler) + +# def check(self, module_test, events): + +# for e in events: +# print(f"{e.type}: {e.data}") + +# web_parameter_extracted = False +# cryptographic_parameter_finding = False +# padding_oracle_detected = False +# for e in events: + +# if e.type == "WEB_PARAMETER": +# if "HTTP Extracted Parameter [encrypted_data] (POST Form" in e.data["description"]: +# web_parameter_extracted = True +# if e.type == "FINDING": +# if ( +# e.data["description"] +# == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [url-encoded]" +# ): +# cryptographic_parameter_finding = True + +# if e.type == "VULNERABILITY": +# if ( +# e.data["description"] +# == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [url-encoded]" +# ): +# padding_oracle_detected = True + +# assert web_parameter_extracted, "Web parameter was not extracted" +# assert cryptographic_parameter_finding, "Cryptographic parameter not detected" +# assert padding_oracle_detected, "Padding oracle vulnerability was not detected" diff --git a/bbot/test/test_step_2/module_tests/test_module_ntlm.py b/bbot/test/test_step_2/module_tests/test_module_ntlm.py index 1e79be770..7b834ef2f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_ntlm.py +++ b/bbot/test/test_step_2/module_tests/test_module_ntlm.py @@ -10,7 +10,8 @@ async def setup_after_prep(self, module_test): request_args = {"uri": "/", "headers": {"test": "header"}} module_test.set_expect_requests(request_args, {}) request_args = { - "uri": "/oab/", "headers": {"Authorization": "NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw=="} + "uri": "/oab/", + "headers": {"Authorization": "NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw=="}, } respond_args = { "headers": { diff --git a/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py b/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py index 812046a66..d16aa6a15 100644 --- a/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py +++ b/bbot/test/test_step_2/module_tests/test_module_reflected_parameters.py @@ -26,7 +26,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) def check(self, module_test, events): - assert any( e.type == "FINDING" and e.data["description"] @@ -39,7 +38,6 @@ class TestReflected_parameters_fromparamminer(TestParamminer_Getparams): modules_overrides = ["httpx", "paramminer_getparams", "reflected_parameters"] def check(self, module_test, events): - assert any( e.type == "FINDING" and "GET Parameter value reflected in response body. Name: [id] Source Module: [paramminer_getparams]" diff --git a/bbot/test/test_step_2/module_tests/test_module_speculate.py b/bbot/test/test_step_2/module_tests/test_module_speculate.py index e40747034..55db777e7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_speculate.py +++ b/bbot/test/test_step_2/module_tests/test_module_speculate.py @@ -63,7 +63,7 @@ def check(self, module_test, events): events_data.add(e.data) assert all( x in events_data - for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") + for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") ) @@ -78,5 +78,5 @@ def check(self, module_test, events): events_data.add(e.data) assert not any( x in events_data - for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") + for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") ) From 81260a3816add4f7ce4a2c1ddbf4f23596d6c602 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 27 Nov 2024 18:16:39 -0500 Subject: [PATCH 217/238] lint --- bbot/modules/internal/dnsresolve.py | 9 +++++++-- bbot/modules/internal/excavate.py | 2 ++ bbot/test/test_step_1/test_web_envelopes.py | 2 +- .../test_step_2/module_tests/test_module_excavate.py | 8 ++++++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index c746b0345..5bb5c5bc4 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -83,9 +83,14 @@ async def handle_event(self, event, **kwargs): event_data_changed = await self.handle_wildcard_event(main_host_event) if event_data_changed: # since data has changed, we check again whether it's a duplicate - if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate(event, add=True): + if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate( + event, add=True + ): if not event._graph_important: - return False, "it's a DNS wildcard, and its module already emitted a similar wildcard event" + return ( + False, + "it's a DNS wildcard, and its module already emitted a similar wildcard event", + ) else: self.debug( f"Event {event} was already emitted by its module, but it's graph-important so it gets a pass" diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 274504e8d..2377a2e6a 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -758,8 +758,10 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte continue if parsed_url.scheme in ["http", "https"]: continue + def abort_if(e): return e.scope_distance > 0 + finding_data = {"host": str(host), "description": f"Non-HTTP URI: {parsed_url.geturl()}"} await self.report(finding_data, event, yara_rule_settings, discovery_context, abort_if=abort_if) protocol_data = {"protocol": parsed_url.scheme, "host": str(host)} diff --git a/bbot/test/test_step_1/test_web_envelopes.py b/bbot/test/test_step_1/test_web_envelopes.py index a5904db14..79da9e829 100644 --- a/bbot/test/test_step_1/test_web_envelopes.py +++ b/bbot/test/test_step_1/test_web_envelopes.py @@ -328,7 +328,7 @@ async def test_web_envelopes(): # null null_envelope = BaseEnvelope.detect("null") assert isinstance(null_envelope, JSONEnvelope) - assert null_envelope.unpacked_data() == None + assert null_envelope.unpacked_data() is None assert null_envelope.pack() == "null" expected_subparams = [([], None)] assert list(null_envelope.get_subparams()) == expected_subparams diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index f3198df11..07ba6f0fb 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -539,7 +539,10 @@ def check(self, module_test, events): if "fit" in e.data["additional_params"].keys(): found_htmltags_img = True - if e.data["description"] == "HTTP Extracted Parameter [blog-post-author-display] (POST Form Submodule)": + if ( + e.data["description"] + == "HTTP Extracted Parameter [blog-post-author-display] (POST Form Submodule)" + ): if e.data["original_value"] == "user.name": if "csrf" in e.data["additional_params"].keys(): found_select_noquotes = True @@ -556,7 +559,8 @@ def check(self, module_test, events): assert found_form_generic_original_value, "Did not extract Form (Generic) parameter original_value" assert found_htmltags_a, "Did not extract parameter(s) from a-tag" assert found_htmltags_img, "Did not extract parameter(s) from img-tag" - assert found_select_noquotes, "Did not extract parameter(s) from select-tag" + assert found_select_noquotes, "Did not extract parameter(s) from select-tag" + class TestExcavateParameterExtraction_postformnoaction(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] From 89effec3489509e5e2c110c1b1ab25fc78cd036c Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 27 Nov 2024 18:20:59 -0500 Subject: [PATCH 218/238] remove dead code --- bbot/core/helpers/web/envelopes.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py index e84be51ba..ddd8f5e97 100644 --- a/bbot/core/helpers/web/envelopes.py +++ b/bbot/core/helpers/web/envelopes.py @@ -291,35 +291,6 @@ def _unpack(self, s): # return s -class BaseSubparamEnvelope(BaseEnvelope): - """ - An envelope that contains subparameters - """ - - end_format = True - - # def get_subparam(self, dot_key=None): - # if dot_key is None: - # dot_key = self.subparam - # data = self.unpacked_data - # for key in dot_key.split("."): - # data = data[key] - # return data - - # def values(self): - # """ - # Returns the values of the unpacked data - - # { - # "key1": "value1", - # "key2": "value2" - # } - # """ - # if isinstance(self.unpacked_data, dict): - # return self.unpacked_data.values() - # return [self.unpacked_data] - - class JSONEnvelope(BaseEnvelope): """ JSON encoding From 93a9eb297a347d76337ba7dbf499948e8fa627dd Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 27 Nov 2024 18:21:33 -0500 Subject: [PATCH 219/238] comment --- bbot/core/helpers/web/envelopes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py index ddd8f5e97..c571a4cfe 100644 --- a/bbot/core/helpers/web/envelopes.py +++ b/bbot/core/helpers/web/envelopes.py @@ -9,7 +9,7 @@ from bbot.core.helpers.misc import is_printable -# TODO: This logic is perfect for extracting params. We should expand it outwards to other higher-level envelopes: +# TODO: This logic is perfect for extracting params. We should expand it outwards to include other higher-level envelopes: # - QueryStringEnvelope # - MultipartFormEnvelope # - HeaderEnvelope From 220323094bb3048a28a8e50564c70844759fa72d Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 27 Nov 2024 19:32:27 -0500 Subject: [PATCH 220/238] fix tests --- bbot/core/event/base.py | 13 ++++++++----- bbot/core/helpers/misc.py | 8 ++++---- bbot/core/helpers/web/envelopes.py | 5 +++-- bbot/test/test_step_1/test_helpers.py | 7 +++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index fc2a35ea2..4f1e3525a 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1342,12 +1342,15 @@ def children(self): children.append(clone) return children - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - original_value = self.data.get("original_value", None) + def sanitize_data(self, data): + original_value = data.get("original_value", None) if original_value is not None: - envelopes = BaseEnvelope.detect(original_value) - setattr(self, "envelopes", envelopes) + try: + envelopes = BaseEnvelope.detect(original_value) + setattr(self, "envelopes", envelopes) + except ValueError as e: + log.verbose(f"Error detecting envelopes for {self}: {e}") + return data def _data_id(self): # dedupe by url:name:param_type diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 73bd45f30..ced61925c 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2869,13 +2869,13 @@ def clean_requirement(req_string): return [clean_requirement(r) for r in dist.requires] -printable_chars = set(string.printable) - - def is_printable(s): """ Check if a string is printable """ if not isinstance(s, str): raise ValueError(f"Expected a string, got {type(s)}") - return set(s) <= printable_chars + + # Exclude control characters that break display/printing + s = set(s) + return all(ord(c) >= 32 or c in "\t\n\r" for c in s) diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py index c571a4cfe..c808c61c2 100644 --- a/bbot/core/helpers/web/envelopes.py +++ b/bbot/core/helpers/web/envelopes.py @@ -120,7 +120,7 @@ def detect(cls, s): if envelope is not False: return envelope del envelope - raise Exception("No envelope detected") + raise Exception(f"No envelope detected for data: '{s}' ({type(s)})") def get_subparams(self, key=None, data=None, recursive=True): if data is None: @@ -266,13 +266,14 @@ class TextEnvelope(BaseEnvelope): # lowest priority means text is the ultimate fallback priority = 10 transparent = True + ignore_exceptions = () def _pack(self, s): return s def _unpack(self, s): if not is_printable(s): - raise Exception("Non-printable data detected in TextEnvelope") + raise ValueError(f"Non-printable data detected in TextEnvelope: '{s}' ({type(s)})") return s diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 2eb67cd13..329994c74 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -460,6 +460,13 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_httpserver): s = "asdf {unused} {used}" assert helpers.safe_format(s, used="fdsa") == "asdf {unused} fdsa" + # is_printable + assert helpers.is_printable("asdf") is True + assert helpers.is_printable(r"""~!@#$^&*()_+=-<>:"?,./;'[]\{}|""") is True + assert helpers.is_printable("ドメイン.テスト") is True + assert helpers.is_printable("4") is True + assert helpers.is_printable("asdf\x00") is False + # punycode assert helpers.smart_encode_punycode("ドメイン.テスト") == "xn--eckwd4c7c.xn--zckzah" assert helpers.smart_decode_punycode("xn--eckwd4c7c.xn--zckzah") == "ドメイン.テスト" From 989d082c58c42b6d3686042f7d3369adf0eeb4db Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 28 Nov 2024 02:30:46 -0500 Subject: [PATCH 221/238] regex and other tweaks --- bbot/core/helpers/regexes.py | 14 +++++++++++--- bbot/modules/internal/excavate.py | 12 ++++++++---- bbot/modules/lightfuzz_submodules/base.py | 6 ++++-- bbot/modules/lightfuzz_submodules/ssti.py | 21 ++++++++++++--------- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index d0a900a0d..6025ae297 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -152,16 +152,24 @@ re.DOTALL, ) generic_form_regex = re.compile( - r"]*\bmethod=)[^>]+(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)[^>]*>([\s\S]*?)<\/form>", re.DOTALL + r"]*\bmethod=)[^>]+(?:\baction=[\"']?([^\s\"'<>]+)[\"']?)[^>]*>([\s\S]*?)<\/form>", re.IGNORECASE | re.DOTALL ) select_tag_regex = re.compile( - r"]+?name=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>(?:\s*]*?value=[\"\']?([_\.\-\w]*)[\"\']?[^>]*>)?" + r"]+?name=[\"\']?([_\-\.\w]+)[\"\']?[^>]*>(?:\s*]*?value=[\"\']?([_\.\-\w]*)[\"\']?[^>]*>)?", re.IGNORECASE | re.DOTALL ) + textarea_tag_regex = re.compile( r']*\bname=["\']?([_\-\.\w]+)["\']?[^>]*>(.*?)', re.IGNORECASE | re.DOTALL ) -tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>") + +button_tag_regex = re.compile( + r"]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([%\-\._=+\/\w]*)[\"\']?[^>]*?>" +) +button_tag_regex2 = re.compile( + r"]*?value=[\"\']?([\-%\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>" +) +tag_attribute_regex = re.compile(r"<[^>]*(?:href|action|src)\s*=\s*[\"\']?([^\s\'\"\>]+)[\"\']?[^>]*>") valid_netloc = r"[^\s!@#$%^&()=/?\\'\";~`<>]+" diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index dbbc36d7b..365b91f38 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -417,7 +417,7 @@ class PostJquery(GetJquery): class HtmlTags(ParameterExtractorRule): name = "HTML Tags" - discovery_regex = r'/<[^>]+(href|src|action)=["\'][^"\']*["\'][^>]*>/ nocase' + discovery_regex = r'/<[^>]+(href|src|action)=["\']?[^"\'>\s]*["\']?[^>]*>/ nocase' extraction_regex = bbot_regexes.tag_attribute_regex output_type = "GETPARAM" @@ -494,6 +494,8 @@ class GetForm(ParameterExtractorRule): "input_tag_novalue_regex": bbot_regexes.input_tag_novalue_regex, "select_tag_regex": bbot_regexes.select_tag_regex, "textarea_tag_regex": bbot_regexes.textarea_tag_regex, + "button_tag_regex": bbot_regexes.button_tag_regex, + "button_tag_regex2": bbot_regexes.button_tag_regex2, } extraction_regex = bbot_regexes.get_form_regex output_type = "GETPARAM" @@ -517,7 +519,7 @@ def extract(self): form_parameters[input_tags[0]] = None else: - if form_content_regex_name == "input_tag_regex2": + if form_content_regex_name in ["input_tag_regex2", "button_tag_regex2"]: input_tags = [(b, a) for a, b in input_tags] for parameter_name, original_value in input_tags: @@ -572,7 +574,7 @@ def __init__(self, excavate): async def process(self, yara_results, event, yara_rule_settings, discovery_context): for identifier, results in yara_results.items(): - for result in results: + for result in results: if identifier not in self.parameterExtractorCallbackDict.keys(): raise ExcavateError("ParameterExtractor YaraRule identified reference non-existent submodule") parameterExtractorSubModule = self.parameterExtractorCallbackDict[identifier]( @@ -601,6 +603,8 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte base_url = ( f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{event.parsed_url.path}" ) + if self.excavate.retain_querystring and len(event.parsed_url.query) > 0: + base_url += f"?{event.parsed_url.query}" url = urljoin(base_url, endpoint) if self.excavate.helpers.validate_parameter(parameter_name, parameter_type): @@ -796,7 +800,7 @@ class URLExtractor(ExcavateRule): tags = "spider-danger" description = "contains tag with src or href attribute" strings: - $url_attr = /<[^>]+(href|src|action)=["\'][^"\']*["\'][^>]*>/ + $url_attr = /<[^>]+(href|src|action)=["\']?[^"\']*["\']?[^>]*>/ condition: $url_attr } diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 9fee502aa..313de8c16 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -149,8 +149,8 @@ async def standard_probe( timeout=10, additional_params_populate_empty=False, speculative_mode="GETPARAM", + allow_redirects=False, ): - probe = self.probe_value_outgoing(probe) if event_type == "SPECULATIVE": @@ -177,6 +177,7 @@ async def standard_probe( json_data = None if event_type == "POSTPARAM": + method = "POST" data = {self.event.data["name"]: probe} if self.event.data["additional_params"] is not None: @@ -203,10 +204,11 @@ async def standard_probe( data=data, json=json_data, url=url, - allow_redirects=False, + allow_redirects=allow_redirects, retries=0, timeout=timeout, ) + def metadata(self): diff --git a/bbot/modules/lightfuzz_submodules/ssti.py b/bbot/modules/lightfuzz_submodules/ssti.py index a4f8df08b..1d1e0fa51 100644 --- a/bbot/modules/lightfuzz_submodules/ssti.py +++ b/bbot/modules/lightfuzz_submodules/ssti.py @@ -3,13 +3,16 @@ class SSTILightfuzz(BaseLightfuzz): async def fuzz(self): + cookies = self.event.data.get("assigned_cookies", {}) - probe_value = "<%25%3d%201337*1337%20%25>" - r = await self.standard_probe(self.event.data["type"], cookies, probe_value) - if r and "1787569" in r.text: - self.results.append( - { - "type": "FINDING", - "description": f"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication]", - } - ) + ssti_probes = ["<%25%3d%201337*1337%20%25>","<%= 1337*1337 %>", "${1337*1337}"] + for probe_value in ssti_probes: + r = await self.standard_probe(self.event.data["type"], cookies, probe_value, allow_redirects=True) + if r and "1787569" in r.text or "1,787,569" in r.text: + self.results.append( + { + "type": "FINDING", + "description": f"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication]", + } + ) + break From d4fbb7335e1de79fdf8f062ce8df820581151342 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 28 Nov 2024 11:30:10 -0500 Subject: [PATCH 222/238] i literally spent FOUR HOURS ON THIS BUG :( :( :( --- bbot/modules/internal/excavate.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 365b91f38..a61d2f8ba 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -525,14 +525,14 @@ def extract(self): for parameter_name, original_value in input_tags: form_parameters[parameter_name] = original_value.strip() - for parameter_name, original_value in form_parameters.items(): - yield ( - self.output_type, - parameter_name, - original_value, - form_action, - _exclude_key(form_parameters, parameter_name), - ) + for parameter_name, original_value in form_parameters.items(): + yield ( + self.output_type, + parameter_name, + original_value, + form_action, + _exclude_key(form_parameters, parameter_name), + ) class GetForm2(GetForm): extraction_regex = bbot_regexes.get_form_regex2 From 2971eb62105010bc9473a48c1afbd35d70560e0d Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 28 Nov 2024 11:30:47 -0500 Subject: [PATCH 223/238] adding default blacklist to lightfuzz presets --- bbot/presets/web/lightfuzz-intense.yml | 6 ++++- bbot/presets/web/lightfuzz-max.yml | 6 ++++- bbot/presets/web/lightfuzz-ssti-test.yml | 28 ++++++++++++++++++++++++ bbot/presets/web/lightfuzz-xss.yml | 6 ++++- bbot/presets/web/lightfuzz.yml | 4 ++++ 5 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 bbot/presets/web/lightfuzz-ssti-test.yml diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml index c63264bf0..f2ceb2522 100644 --- a/bbot/presets/web/lightfuzz-intense.yml +++ b/bbot/presets/web/lightfuzz-intense.yml @@ -23,4 +23,8 @@ config: enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss] disable_post: False excavate: - retain_querystring: True \ No newline at end of file + retain_querystring: True + +blacklist: + # Prevent spider from invalidating sessions by logging out + - "RE:/.*(sign|log)[_-]?out" \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-max.yml b/bbot/presets/web/lightfuzz-max.yml index 8bb5f51de..6a7530c32 100644 --- a/bbot/presets/web/lightfuzz-max.yml +++ b/bbot/presets/web/lightfuzz-max.yml @@ -23,4 +23,8 @@ config: enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss] disable_post: False excavate: - retain_querystring: True \ No newline at end of file + retain_querystring: True + +blacklist: + # Prevent spider from invalidating sessions by logging out + - "RE:/.*(sign|log)[_-]?out" \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz-ssti-test.yml b/bbot/presets/web/lightfuzz-ssti-test.yml new file mode 100644 index 000000000..1b7674d98 --- /dev/null +++ b/bbot/presets/web/lightfuzz-ssti-test.yml @@ -0,0 +1,28 @@ +description: Discovery web parameters and lightly fuzz them for vulnerabilities, with more intense discovery techniques + +modules: + - httpx + - lightfuzz + - robots + - badsecrets + - hunt + - reflected_parameters + +blacklist: + # Prevent spider from invalidating sessions by logging out + - "RE:/.*(sign|log)[_-]?out" + +config: + url_querystring_remove: False + url_querystring_collapse: True + web: + spider_distance: 6 + spider_depth: 7 + modules: + lightfuzz: + force_common_headers: False + enabled_submodules: [ssti] + disable_post: False + excavate: + retain_querystring: True + diff --git a/bbot/presets/web/lightfuzz-xss.yml b/bbot/presets/web/lightfuzz-xss.yml index 493317496..d3c9755ee 100644 --- a/bbot/presets/web/lightfuzz-xss.yml +++ b/bbot/presets/web/lightfuzz-xss.yml @@ -14,4 +14,8 @@ config: modules: lightfuzz: enabled_submodules: [xss] - disable_post: True \ No newline at end of file + disable_post: True + +blacklist: + # Prevent spider from invalidating sessions by logging out + - "RE:/.*(sign|log)[_-]?out" \ No newline at end of file diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml index e0b674075..1c48c668b 100644 --- a/bbot/presets/web/lightfuzz.yml +++ b/bbot/presets/web/lightfuzz.yml @@ -20,3 +20,7 @@ config: disable_post: True excavate: retain_querystring: True + +blacklist: + # Prevent spider from invalidating sessions by logging out + - "RE:/.*(sign|log)[_-]?out" \ No newline at end of file From 0510ee0338a859b034ca8dfc31453c22d7bc076a Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 28 Nov 2024 11:32:08 -0500 Subject: [PATCH 224/238] ssti submodule tweak --- bbot/modules/lightfuzz_submodules/ssti.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/ssti.py b/bbot/modules/lightfuzz_submodules/ssti.py index 1d1e0fa51..53c7605fa 100644 --- a/bbot/modules/lightfuzz_submodules/ssti.py +++ b/bbot/modules/lightfuzz_submodules/ssti.py @@ -5,14 +5,14 @@ class SSTILightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - ssti_probes = ["<%25%3d%201337*1337%20%25>","<%= 1337*1337 %>", "${1337*1337}"] + ssti_probes = ["<%25%3d%201337*1337%20%25>","<%= 1337*1337 %>", "${1337*1337}", "%24%7b1337*1337%7d"] for probe_value in ssti_probes: - r = await self.standard_probe(self.event.data["type"], cookies, probe_value, allow_redirects=True) - if r and "1787569" in r.text or "1,787,569" in r.text: + r = await self.standard_probe(self.event.data["type"], cookies, probe_value, allow_redirects=True) + if r and ("1787569" in r.text or "1,787,569" in r.text): self.results.append( { "type": "FINDING", - "description": f"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication]", + "description": f"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication] Payload: [{probe_value}]", } ) break From 060babfdbf30c38313e21e7cb73019ce469d6c06 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 28 Nov 2024 11:49:46 -0500 Subject: [PATCH 225/238] may this bug be vanquished for all of eternity --- .../module_tests/test_module_excavate.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index 447b35f18..67fb43ce1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -592,6 +592,47 @@ def check(self, module_test, events): assert excavate_getparam_extraction, "Excavate failed to extract web parameter" +class TestExcavateParameterExtraction_additionalparams(ModuleTestBase): + + targets = ["http://127.0.0.1:8888/"] + + # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER + modules_overrides = ["httpx", "excavate", "hunt"] + postformnoaction_extract_multiparams_html = """ + +

Post for without action

+
+ + + + +
+ + """ + + async def setup_after_prep(self, module_test): + respond_args = {"response_data": self.postformnoaction_extract_multiparams_html, "headers": {"Content-Type": "text/html"}} + module_test.set_expect_requests(respond_args=respond_args) + + def check(self, module_test, events): + + excavate_additionalparam_extraction_param1 = False + excavate_additionalparam_extraction_param2 = False + excavate_additionalparam_extraction_param3 = False + for e in events: + if e.type == "WEB_PARAMETER": + if e.data["name"] == "template-action" and "csrf" in e.data["additional_params"].keys() and "template" in e.data["additional_params"].keys(): + excavate_additionalparam_extraction_param1 = True + if e.data["name"] == "template" and "csrf" in e.data["additional_params"].keys() and "template-action" in e.data["additional_params"].keys(): + excavate_additionalparam_extraction_param2 = True + if e.data["name"] == "csrf" and "template" in e.data["additional_params"].keys() and "template-action" in e.data["additional_params"].keys(): + excavate_additionalparam_extraction_param3 = True + assert excavate_additionalparam_extraction_param1, "Excavate failed to extract web parameter with correct additional data (param 1)" + assert excavate_additionalparam_extraction_param2, "Excavate failed to extract web parameter with correct additional data (param 2)" + assert excavate_additionalparam_extraction_param3, "Excavate failed to extract web parameter with correct additional data (param 3)" + class TestExcavateParameterExtraction_getparam(ModuleTestBase): targets = ["http://127.0.0.1:8888/"] From 524308922af967b57e60ca85ee94f8a28ce17b24 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 29 Nov 2024 08:45:20 -0500 Subject: [PATCH 226/238] additional ssti signature --- bbot/modules/lightfuzz_submodules/ssti.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/ssti.py b/bbot/modules/lightfuzz_submodules/ssti.py index 53c7605fa..766356b58 100644 --- a/bbot/modules/lightfuzz_submodules/ssti.py +++ b/bbot/modules/lightfuzz_submodules/ssti.py @@ -5,7 +5,7 @@ class SSTILightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - ssti_probes = ["<%25%3d%201337*1337%20%25>","<%= 1337*1337 %>", "${1337*1337}", "%24%7b1337*1337%7d"] + ssti_probes = ["<%25%3d%201337*1337%20%25>","<%= 1337*1337 %>", "${1337*1337}", "%24%7b1337*1337%7d", "1,787{{z}},569"] for probe_value in ssti_probes: r = await self.standard_probe(self.event.data["type"], cookies, probe_value, allow_redirects=True) if r and ("1787569" in r.text or "1,787,569" in r.text): @@ -15,4 +15,4 @@ async def fuzz(self): "description": f"POSSIBLE Server-side Template Injection. {self.metadata()} Detection Method: [Integer Multiplication] Payload: [{probe_value}]", } ) - break + break \ No newline at end of file From aa1bbcc213f1eb83be52e366e81d55242c4d4193 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 29 Nov 2024 13:04:24 -0500 Subject: [PATCH 227/238] adding new envelope multiple test --- .../module_tests/test_module_lightfuzz.py | 292 ++++++++++-------- 1 file changed, 162 insertions(+), 130 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 4cfac4d39..ea6a3081d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -362,6 +362,38 @@ def request_handler(self, request): return Response(parameter_block, status=200) +# Base64 (JSON) Multiple Envelope Detection +class Test_Lightfuzz_envelope_multiple_json(Test_Lightfuzz_envelope_base64): + def request_handler(self, request): + qs = str(request.query_string.decode()) + + parameter_block = """ +
+
+ + +
+
+ """ + return Response(parameter_block, status=200) + + + def check(self, module_test, events): + web_parameter_emitted = False + web_parameter_clone_emitted = False + + for e in events: + if e.type == "WEB_PARAMETER": + for subparam in e.envelopes.get_subparams(): + if len(subparam[0]) > 0: + if subparam[0][0] == "string1" and subparam[1] == "value1": + web_parameter_emitted = True + if subparam[0][0] == "string2" and subparam[1] == "value2": + web_parameter_clone_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert web_parameter_clone_emitted, "WEB_PARAMETER clone was not emitted" + # Base64 (XML) Envelope XSS Detection class Test_Lightfuzz_envelope_xmlb64(Test_Lightfuzz_envelope_base64): @@ -1189,59 +1221,59 @@ def check(self, module_test, events): assert cmdi_interacttsh_finding_emitted, "interactsh CMDi FINDING not emitted" -# class Test_Lightfuzz_speculative(ModuleTestBase): -# targets = ["http://127.0.0.1:8888/"] -# modules_overrides = ["httpx", "excavate", "paramminer_getparams", "lightfuzz"] -# config_overrides = { -# "interactsh_disable": True, -# "modules": { -# "lightfuzz": {"enabled_submodules": ["xss"]}, -# "paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}, -# }, -# } - -# def request_handler(self, request): - -# qs = str(request.query_string.decode()) -# parameter_block = """ -# { -# "search": 1, -# "common": 1 -# } -# """ -# if "search=" in qs: -# value = qs.split("=")[1] -# if "&" in value: -# value = value.split("&")[0] -# xss_block = f""" -#
-#

0 search results for '{unquote(value)}'

-#
-#
-# """ -# return Response(xss_block, status=200) -# return Response(parameter_block, status=200, headers={"Content-Type": "application/json"}) - -# async def setup_after_prep(self, module_test): -# module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" -# expect_args = re.compile("/") -# module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) - -# def check(self, module_test, events): -# excavate_json_extraction = False -# xss_finding_emitted = False - -# for e in events: -# if e.type == "WEB_PARAMETER": -# if "HTTP Extracted Parameter (speculative from json content) [search]" in e.data["description"]: -# excavate_json_extraction = True - -# if e.type == "FINDING": -# if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags" in e.data["description"]: -# xss_finding_emitted = True - -# assert excavate_json_extraction, "Excavate failed to extract json parameter" -# assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" +class Test_Lightfuzz_speculative(ModuleTestBase): + targets = ["http://127.0.0.1:8888/"] + modules_overrides = ["httpx", "excavate", "paramminer_getparams", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": {"enabled_submodules": ["xss"]}, + "paramminer_getparams": {"wordlist": tempwordlist([]), "recycle_words": True}, + }, + } + + def request_handler(self, request): + + qs = str(request.query_string.decode()) + parameter_block = """ + { + "search": 1, + "common": 1 + } + """ + if "search=" in qs: + value = qs.split("=")[1] + if "&" in value: + value = value.split("&")[0] + xss_block = f""" +
+

0 search results for '{unquote(value)}'

+
+
+ """ + return Response(xss_block, status=200) + return Response(parameter_block, status=200, headers={"Content-Type": "application/json"}) + + async def setup_after_prep(self, module_test): + module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA" + expect_args = re.compile("/") + module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler) + + def check(self, module_test, events): + excavate_json_extraction = False + xss_finding_emitted = False + + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter (speculative from json content) [search]" in e.data["description"]: + excavate_json_extraction = True + + if e.type == "FINDING": + if "Possible Reflected XSS. Parameter: [search] Context: [Between Tags" in e.data["description"]: + xss_finding_emitted = True + + assert excavate_json_extraction, "Excavate failed to extract json parameter" + assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" class Test_Lightfuzz_crypto_error(ModuleTestBase): @@ -1345,80 +1377,80 @@ def check(self, module_test, events): ), "Crypto Error Message FINDING was emitted (it is an intentional false positive)" -# class Test_PaddingOracleDetection(ModuleTestBase): - -# targets = ["http://127.0.0.1:8888"] -# modules_overrides = ["httpx", "excavate", "lightfuzz"] -# config_overrides = { -# "interactsh_disable": True, -# "modules": { -# "lightfuzz": { -# "enabled_submodules": ["crypto"], -# } -# }, -# } - -# def request_handler(self, request): -# encrypted_value = quote( -# "dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==" -# ) -# default_html_response = f""" -# -# -#
-# -# -#
-# -# -# """ - -# if "/decrypt" in request.url and request.method == "POST": -# if request.form and request.form["encrypted_data"]: -# encrypted_data = request.form["encrypted_data"] -# if "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm" in encrypted_data: -# response_content = "Padding error detected" -# elif "4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg" in encrypted_data: -# response_content = "DIFFERENT CRYPTOGRAPHIC ERROR" -# elif "AAAAAAA" in encrypted_data: -# response_content = "YET DIFFERENT CRYPTOGRAPHIC ERROR" -# else: -# response_content = "Decryption failed" - -# return Response(response_content, status=200) -# else: -# return Response(default_html_response, status=200) - -# async def setup_after_prep(self, module_test): -# module_test.set_expect_requests_handler(expect_args=re.compile(".*"), request_handler=self.request_handler) - -# def check(self, module_test, events): - -# for e in events: -# print(f"{e.type}: {e.data}") - -# web_parameter_extracted = False -# cryptographic_parameter_finding = False -# padding_oracle_detected = False -# for e in events: - -# if e.type == "WEB_PARAMETER": -# if "HTTP Extracted Parameter [encrypted_data] (POST Form" in e.data["description"]: -# web_parameter_extracted = True -# if e.type == "FINDING": -# if ( -# e.data["description"] -# == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [url-encoded]" -# ): -# cryptographic_parameter_finding = True - -# if e.type == "VULNERABILITY": -# if ( -# e.data["description"] -# == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [url-encoded]" -# ): -# padding_oracle_detected = True - -# assert web_parameter_extracted, "Web parameter was not extracted" -# assert cryptographic_parameter_finding, "Cryptographic parameter not detected" -# assert padding_oracle_detected, "Padding oracle vulnerability was not detected" +class Test_PaddingOracleDetection(ModuleTestBase): + + targets = ["http://127.0.0.1:8888"] + modules_overrides = ["httpx", "excavate", "lightfuzz"] + config_overrides = { + "interactsh_disable": True, + "modules": { + "lightfuzz": { + "enabled_submodules": ["crypto"], + } + }, + } + + def request_handler(self, request): + encrypted_value = quote( + "dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q+4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg==" + ) + default_html_response = f""" + + +
+ + +
+ + + """ + + if "/decrypt" in request.url and request.method == "POST": + if request.form and request.form["encrypted_data"]: + encrypted_data = request.form["encrypted_data"] + if "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAALwAgLKWJi2nWKbh9ag5rnhm" in encrypted_data: + response_content = "Padding error detected" + elif "4GXVGZbo0DTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg" in encrypted_data: + response_content = "DIFFERENT CRYPTOGRAPHIC ERROR" + elif "AAAAAAA" in encrypted_data: + response_content = "YET DIFFERENT CRYPTOGRAPHIC ERROR" + else: + response_content = "Decryption failed" + + return Response(response_content, status=200) + else: + return Response(default_html_response, status=200) + + async def setup_after_prep(self, module_test): + module_test.set_expect_requests_handler(expect_args=re.compile(".*"), request_handler=self.request_handler) + + def check(self, module_test, events): + + for e in events: + print(f"{e.type}: {e.data}") + + web_parameter_extracted = False + cryptographic_parameter_finding = False + padding_oracle_detected = False + for e in events: + + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [encrypted_data] (POST Form" in e.data["description"]: + web_parameter_extracted = True + if e.type == "FINDING": + if ( + e.data["description"] + == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [url-encoded]" + ): + cryptographic_parameter_finding = True + + if e.type == "VULNERABILITY": + if ( + e.data["description"] + == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [url-encoded]" + ): + padding_oracle_detected = True + + assert web_parameter_extracted, "Web parameter was not extracted" + assert cryptographic_parameter_finding, "Cryptographic parameter not detected" + assert padding_oracle_detected, "Padding oracle vulnerability was not detected" From 737999b9d2d326d6fd9bbffbcc7d5d345e52fbdf Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 29 Nov 2024 14:40:17 -0500 Subject: [PATCH 228/238] envelope bug fixes / enhancements --- bbot/core/event/base.py | 17 ++++++----- bbot/core/helpers/web/envelopes.py | 30 +++++++++++++++++-- bbot/modules/lightfuzz.py | 2 +- bbot/modules/lightfuzz_submodules/base.py | 24 +++++++-------- bbot/modules/lightfuzz_submodules/crypto.py | 8 +++-- bbot/modules/lightfuzz_submodules/path.py | 2 +- bbot/modules/lightfuzz_submodules/serial.py | 2 +- bbot/modules/lightfuzz_submodules/sqli.py | 2 +- .../module_tests/test_module_lightfuzz.py | 6 ++-- 9 files changed, 59 insertions(+), 34 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 4f1e3525a..74ec0b696 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1331,15 +1331,18 @@ def children(self): children = [] envelopes = getattr(self, "envelopes", None) if envelopes is not None: - subparams = list(self.envelopes.get_subparams()) + subparams = sorted(list(self.envelopes.get_subparams())) if envelopes.selected_subparam is None: - for subparam, _ in subparams: - clone = self.clone() - clone.envelopes = deepcopy(envelopes) - clone.envelopes.selected_subparam = subparam - clone.parent = self - children.append(clone) + current_subparam = subparams[0] + envelopes.selected_subparam = current_subparam[0] + if len(subparams) > 1: + for subparam, _ in subparams[1:]: + clone = self.clone() + clone.envelopes = deepcopy(envelopes) + clone.envelopes.selected_subparam = subparam + clone.parent = self + children.append(clone) return children def sanitize_data(self, data): diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py index c808c61c2..c000c5d1c 100644 --- a/bbot/core/helpers/web/envelopes.py +++ b/bbot/core/helpers/web/envelopes.py @@ -71,6 +71,13 @@ def final_envelope(self): return self.unpacked_data(recursive=False).final_envelope except AttributeError: return self + + @property + def friendly_name(self): + if self.friendly_name: + return self.friendly_name + else: + return self.name def pack(self, data=None): if data is None: @@ -194,15 +201,24 @@ def num_envelopes(self): def summary(self): if self.transparent: return "" - self_string = f"{self.name}" - if self.selected_subparam: - self_string += f" [{self.selected_subparam}]" + self_string = f"{self.friendly_name}" with suppress(AttributeError): child_envelope = self.unpacked_data(recursive=False) child_summary = child_envelope.summary if child_summary: self_string += f" -> {child_summary}" + + if self.selected_subparam: + self_string += f" [{'.'.join(self.selected_subparam)}]" return self_string + + def to_dict(self): + return self.summary + + def __str__(self): + return self.summary + + __repr__ = __str__ class HexEnvelope(BaseEnvelope): @@ -210,6 +226,8 @@ class HexEnvelope(BaseEnvelope): Hexadecimal encoding """ + friendly_name = "Hexadecimal-Encoded" + ignore_exceptions = (ValueError, UnicodeDecodeError) def _pack(self, s): @@ -224,6 +242,8 @@ class B64Envelope(BaseEnvelope): Base64 encoding """ + friendly_name = "Base64-Encoded" + ignore_exceptions = (binascii.Error, UnicodeDecodeError, ValueError) def unpack(self, s): @@ -244,6 +264,8 @@ class URLEnvelope(BaseEnvelope): URL encoding """ + friendly_name = "URL-Encoded" + def unpack(self, s): unpacked = super().unpack(s) if unpacked == s: @@ -297,6 +319,7 @@ class JSONEnvelope(BaseEnvelope): JSON encoding """ + friendly_name = "JSON-formatted" end_format = True priority = 8 ignore_exceptions = (json.JSONDecodeError,) @@ -313,6 +336,7 @@ class XMLEnvelope(BaseEnvelope): XML encoding """ + friendly_name = "XML-formatted" end_format = True priority = 9 ignore_exceptions = (ExpatError,) diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py index 8ddf07e04..ee3f04b20 100644 --- a/bbot/modules/lightfuzz.py +++ b/bbot/modules/lightfuzz.py @@ -133,7 +133,7 @@ async def run_submodule(self, submodule, event): envelope_summary = getattr(envelopes, "summary", None) if envelope_summary: # Append the envelope summary to the description - event_data["description"] += f" Envelopes: {envelope_summary}" + event_data["description"] += f" Envelopes: [{envelope_summary}]" if r["type"] == "VULNERABILITY": event_data["severity"] = r["severity"] diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index cdac02198..521cc1858 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -19,7 +19,7 @@ def additional_params_process(self, additional_params, additional_params_populat return new_additional_params async def send_probe(self, probe): - probe = self.probe_value_outgoing(probe) + probe = self.outgoing_probe_value(probe) getparams = {self.event.data["name"]: probe} url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl() self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}") @@ -30,7 +30,7 @@ async def send_probe(self, probe): def compare_baseline( self, event_type, probe, cookies, additional_params_populate_empty=False, speculative_mode="GETPARAM" ): - probe = self.probe_value_outgoing(probe) + probe = self.outgoing_probe_value(probe) http_compare = None if event_type == "SPECULATIVE": @@ -104,7 +104,7 @@ async def compare_probe( additional_params_override={}, speculative_mode="GETPARAM", ): - probe = self.probe_value_outgoing(probe) + probe = self.outgoing_probe_value(probe) additional_params = copy.deepcopy(self.event.data.get("additional_params", {})) if additional_params_override: for k, v in additional_params_override.items(): @@ -150,7 +150,7 @@ async def standard_probe( speculative_mode="GETPARAM", allow_redirects=False, ): - probe = self.probe_value_outgoing(probe) + probe = self.outgoing_probe_value(probe) if event_type == "SPECULATIVE": event_type = speculative_mode @@ -217,32 +217,28 @@ def metadata(self): ) return metadata_string - def probe_value_incoming(self, populate_empty=True): + def incoming_probe_value(self, populate_empty=True): envelopes = getattr(self.event, "envelopes", None) probe_value = "" if envelopes is not None: probe_value = envelopes.get_subparam() + self.lightfuzz.debug(f"incoming_probe_value (after unpacking): {probe_value} with envelopes [{envelopes}]") if not probe_value: if populate_empty is True: probe_value = self.lightfuzz.helpers.rand_string(10, numeric_only=True) else: probe_value = "" - self.lightfuzz.debug(f"probe_value_incoming (before unpacking): {probe_value}") - if envelopes is not None: - envelopes.set_subparam(value=probe_value) - probe_value = envelopes.pack() - self.lightfuzz.debug(f"probe_value_incoming (after unpacking): {probe_value}") if not isinstance(probe_value, str): raise ValueError( - f"probe_value_incoming should always be a string (got {type(probe_value)} / {probe_value})" + f"incoming_probe_value should always be a string (got {type(probe_value)} / {probe_value})" ) return probe_value - def probe_value_outgoing(self, outgoing_probe_value): - self.lightfuzz.debug(f"probe_value_outgoing (before packing): {outgoing_probe_value} / {self.event}") + def outgoing_probe_value(self, outgoing_probe_value): + self.lightfuzz.debug(f"outgoing_probe_value (before packing): {outgoing_probe_value} / {self.event}") envelopes = getattr(self.event, "envelopes", None) if envelopes is not None: envelopes.set_subparam(value=outgoing_probe_value) outgoing_probe_value = envelopes.pack() - self.lightfuzz.debug(f"probe_value_outgoing (after packing): {outgoing_probe_value} / {self.event}") + self.lightfuzz.debug(f"outgoing_probe_value (after packing): {outgoing_probe_value} with envelopes [{envelopes}] / {self.event}") return outgoing_probe_value diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index cda98b1dd..096646aa3 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -75,6 +75,8 @@ def format_agnostic_encode(data, encoding, urlencode=False): def modify_string(input_string, action="truncate", position=None, extension_length=1): if not isinstance(input_string, str): input_string = str(input_string) + print("#####") + print(input_string) data, encoding = CryptoLightfuzz.format_agnostic_decode(input_string) if encoding != "base64" and encoding != "hex": @@ -232,8 +234,10 @@ def identify_hash_function(hash_bytes): return hash_functions[hash_length] async def fuzz(self): + cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value_incoming(populate_empty=False) + probe_value = self.incoming_probe_value(populate_empty=False) + if not probe_value: self.lightfuzz.debug( f"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" @@ -250,7 +254,7 @@ async def fuzz(self): mutate_probe_value = self.modify_string(probe_value, action="mutate") except ValueError as e: self.lightfuzz.debug( - f"Encountered error modifying value for parameter {self.event.data['name']}: {e} , aborting" + f"Encountered error modifying value for parameter [{self.event.data['name']}]: {e} , aborting" ) return diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py index f39ef93f4..827af65f2 100644 --- a/bbot/modules/lightfuzz_submodules/path.py +++ b/bbot/modules/lightfuzz_submodules/path.py @@ -8,7 +8,7 @@ class PathTraversalLightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value_incoming(populate_empty=False) + probe_value = self.incoming_probe_value(populate_empty=False) if not probe_value: self.lightfuzz.debug( f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" diff --git a/bbot/modules/lightfuzz_submodules/serial.py b/bbot/modules/lightfuzz_submodules/serial.py index a45940d18..e6cf2da76 100644 --- a/bbot/modules/lightfuzz_submodules/serial.py +++ b/bbot/modules/lightfuzz_submodules/serial.py @@ -26,7 +26,7 @@ async def fuzz(self): "java.io.optionaldataexception", ] - probe_value = self.probe_value_incoming(populate_empty=False) + probe_value = self.incoming_probe_value(populate_empty=False) if probe_value: self.lightfuzz.debug( f"The Serialization Submodule only operates when there if no original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]" diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py index 32c3f45fb..1f7d677cc 100644 --- a/bbot/modules/lightfuzz_submodules/sqli.py +++ b/bbot/modules/lightfuzz_submodules/sqli.py @@ -39,7 +39,7 @@ def evaluate_delay(self, mean_baseline, measured_delay): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value_incoming(populate_empty=True) + probe_value = self.incoming_probe_value(populate_empty=True) http_compare = self.compare_baseline( self.event.data["type"], probe_value, cookies, additional_params_populate_empty=True ) diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index ea6a3081d..4ff3a478c 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -365,7 +365,6 @@ def request_handler(self, request): # Base64 (JSON) Multiple Envelope Detection class Test_Lightfuzz_envelope_multiple_json(Test_Lightfuzz_envelope_base64): def request_handler(self, request): - qs = str(request.query_string.decode()) parameter_block = """
@@ -1425,7 +1424,6 @@ async def setup_after_prep(self, module_test): module_test.set_expect_requests_handler(expect_args=re.compile(".*"), request_handler=self.request_handler) def check(self, module_test, events): - for e in events: print(f"{e.type}: {e.data}") @@ -1440,14 +1438,14 @@ def check(self, module_test, events): if e.type == "FINDING": if ( e.data["description"] - == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [url-encoded]" + == "Probable Cryptographic Parameter. Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Detection Technique(s): [Single-byte Mutation, Data Truncation] Envelopes: [URL-Encoded]" ): cryptographic_parameter_finding = True if e.type == "VULNERABILITY": if ( e.data["description"] - == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [url-encoded]" + == "Padding Oracle Vulnerability. Block size: [16] Parameter: [encrypted_data] Parameter Type: [POSTPARAM] Original Value: [dplyorsu8VUriMW/8DqVDU6kRwL/FDk3Q%2B4GXVGZbo0CTh9YX1YvzZZJrYe4cHxvAICyliYtp1im4fWoOa54Zg%3D%3D] Envelopes: [URL-Encoded]" ): padding_oracle_detected = True From c3ec029338b4c9a1b8c601e712befccdb98b2055 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 29 Nov 2024 14:42:10 -0500 Subject: [PATCH 229/238] remove debug --- bbot/modules/lightfuzz_submodules/crypto.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py index 096646aa3..3ebb3bacd 100644 --- a/bbot/modules/lightfuzz_submodules/crypto.py +++ b/bbot/modules/lightfuzz_submodules/crypto.py @@ -75,8 +75,6 @@ def format_agnostic_encode(data, encoding, urlencode=False): def modify_string(input_string, action="truncate", position=None, extension_length=1): if not isinstance(input_string, str): input_string = str(input_string) - print("#####") - print(input_string) data, encoding = CryptoLightfuzz.format_agnostic_decode(input_string) if encoding != "base64" and encoding != "hex": From cbfae579e4066ac887c2b38c084b5a7d2149ba94 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 29 Nov 2024 19:08:29 -0500 Subject: [PATCH 230/238] variable name --- bbot/modules/lightfuzz_submodules/cmdi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py index 450697a8d..57acfdbb5 100644 --- a/bbot/modules/lightfuzz_submodules/cmdi.py +++ b/bbot/modules/lightfuzz_submodules/cmdi.py @@ -7,7 +7,7 @@ class CmdILightfuzz(BaseLightfuzz): async def fuzz(self): cookies = self.event.data.get("assigned_cookies", {}) - probe_value = self.probe_value_incoming() + probe_value = self.incoming_probe_value() canary = self.lightfuzz.helpers.rand_string(10, numeric_only=True) http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies) From c8888c0ecaa3d2ebdaa3dd6a6c6bc26e5d512792 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 30 Nov 2024 11:06:18 -0500 Subject: [PATCH 231/238] stringifying all original values --- bbot/modules/internal/excavate.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 1ba79018c..bb993828c 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -615,7 +615,7 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte "host": parsed_url.hostname, "type": parameter_type, "name": parameter_name, - "original_value": original_value, + "original_value": str(original_value), "url": self.excavate.url_unparse(parameter_type, parsed_url), "additional_params": additional_params, "assigned_cookies": self.excavate.assigned_cookies, @@ -1025,7 +1025,7 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon "host": str(event.host), "type": "SPECULATIVE", "name": parameter_name, - "original_value": original_value, + "original_value": str(original_value), "url": str(event.data["url"]), "additional_params": {}, "assigned_cookies": self.assigned_cookies, @@ -1083,7 +1083,7 @@ async def handle_event(self, event): "host": parsed_url.hostname, "type": "GETPARAM", "name": parameter_name, - "original_value": original_value, + "original_value": str(original_value), "url": self.url_unparse("GETPARAM", parsed_url), "description": description, "additional_params": additional_params, @@ -1100,7 +1100,7 @@ async def handle_event(self, event): "host": event.parsed_url.hostname, "type": "COOKIE", "name": custom_cookie_name, - "original_value": custom_cookie_value, + "original_value": str(custom_cookie_value), "url": self.url_unparse("COOKIE", event.parsed_url), "description": description, "additional_params": _exclude_key(custom_cookies, custom_cookie_name), @@ -1117,7 +1117,7 @@ async def handle_event(self, event): "host": event.parsed_url.hostname, "type": "HEADER", "name": custom_header_name, - "original_value": custom_header_value, + "original_value": str(custom_header_value), "url": self.url_unparse("HEADER", event.parsed_url), "description": description, "additional_params": _exclude_key(custom_headers, custom_header_name), @@ -1156,7 +1156,7 @@ async def handle_event(self, event): "host": str(event.host), "type": "COOKIE", "name": cookie_name, - "original_value": cookie_value, + "original_value": str(cookie_value), "url": self.url_unparse("COOKIE", event.parsed_url), "description": description, } @@ -1199,7 +1199,7 @@ async def handle_event(self, event): "host": parsed_url.hostname, "type": "GETPARAM", "name": parameter_name, - "original_value": original_value, + "original_value": str(original_value), "url": self.url_unparse("GETPARAM", parsed_url), "description": description, "additional_params": additional_params, From 063a34d80d847a8272c17b1768f782f8489e6be7 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Sat, 30 Nov 2024 11:47:21 -0500 Subject: [PATCH 232/238] Revert "stringifying all original values" This reverts commit c8888c0ecaa3d2ebdaa3dd6a6c6bc26e5d512792. --- bbot/modules/internal/excavate.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index bb993828c..1ba79018c 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -615,7 +615,7 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte "host": parsed_url.hostname, "type": parameter_type, "name": parameter_name, - "original_value": str(original_value), + "original_value": original_value, "url": self.excavate.url_unparse(parameter_type, parsed_url), "additional_params": additional_params, "assigned_cookies": self.excavate.assigned_cookies, @@ -1025,7 +1025,7 @@ async def search(self, data, event, content_type, discovery_context="HTTP respon "host": str(event.host), "type": "SPECULATIVE", "name": parameter_name, - "original_value": str(original_value), + "original_value": original_value, "url": str(event.data["url"]), "additional_params": {}, "assigned_cookies": self.assigned_cookies, @@ -1083,7 +1083,7 @@ async def handle_event(self, event): "host": parsed_url.hostname, "type": "GETPARAM", "name": parameter_name, - "original_value": str(original_value), + "original_value": original_value, "url": self.url_unparse("GETPARAM", parsed_url), "description": description, "additional_params": additional_params, @@ -1100,7 +1100,7 @@ async def handle_event(self, event): "host": event.parsed_url.hostname, "type": "COOKIE", "name": custom_cookie_name, - "original_value": str(custom_cookie_value), + "original_value": custom_cookie_value, "url": self.url_unparse("COOKIE", event.parsed_url), "description": description, "additional_params": _exclude_key(custom_cookies, custom_cookie_name), @@ -1117,7 +1117,7 @@ async def handle_event(self, event): "host": event.parsed_url.hostname, "type": "HEADER", "name": custom_header_name, - "original_value": str(custom_header_value), + "original_value": custom_header_value, "url": self.url_unparse("HEADER", event.parsed_url), "description": description, "additional_params": _exclude_key(custom_headers, custom_header_name), @@ -1156,7 +1156,7 @@ async def handle_event(self, event): "host": str(event.host), "type": "COOKIE", "name": cookie_name, - "original_value": str(cookie_value), + "original_value": cookie_value, "url": self.url_unparse("COOKIE", event.parsed_url), "description": description, } @@ -1199,7 +1199,7 @@ async def handle_event(self, event): "host": parsed_url.hostname, "type": "GETPARAM", "name": parameter_name, - "original_value": str(original_value), + "original_value": original_value, "url": self.url_unparse("GETPARAM", parsed_url), "description": description, "additional_params": additional_params, From 64c6943da31ed880012d1b777713c57ef5414163 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Tue, 3 Dec 2024 10:05:50 -0500 Subject: [PATCH 233/238] ensuring original_value is str --- bbot/core/helpers/web/envelopes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py index c000c5d1c..093c036a6 100644 --- a/bbot/core/helpers/web/envelopes.py +++ b/bbot/core/helpers/web/envelopes.py @@ -143,7 +143,7 @@ def get_subparams(self, key=None, data=None, recursive=True): else: yield full_key, v else: - yield [], data + yield [], str(data) def get_subparam(self, key=None, recursive=True): if key is None: @@ -160,7 +160,7 @@ def get_subparam(self, key=None, recursive=True): else: for segment in key: data = data[segment] - return data + return str(data) def set_subparam(self, key=None, value=None, recursive=True): envelope = self From a885451fc01c2a97cd1a953bc0ae693cabdf1d5e Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 14:40:37 -0500 Subject: [PATCH 234/238] fix? --- bbot/core/helpers/web/envelopes.py | 4 ++-- bbot/modules/lightfuzz_submodules/base.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bbot/core/helpers/web/envelopes.py b/bbot/core/helpers/web/envelopes.py index 093c036a6..c000c5d1c 100644 --- a/bbot/core/helpers/web/envelopes.py +++ b/bbot/core/helpers/web/envelopes.py @@ -143,7 +143,7 @@ def get_subparams(self, key=None, data=None, recursive=True): else: yield full_key, v else: - yield [], str(data) + yield [], data def get_subparam(self, key=None, recursive=True): if key is None: @@ -160,7 +160,7 @@ def get_subparam(self, key=None, recursive=True): else: for segment in key: data = data[segment] - return str(data) + return data def set_subparam(self, key=None, value=None, recursive=True): envelope = self diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 521cc1858..32d6d8033 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -228,10 +228,11 @@ def incoming_probe_value(self, populate_empty=True): probe_value = self.lightfuzz.helpers.rand_string(10, numeric_only=True) else: probe_value = "" - if not isinstance(probe_value, str): - raise ValueError( - f"incoming_probe_value should always be a string (got {type(probe_value)} / {probe_value})" - ) + # if not isinstance(probe_value, str): + # raise ValueError( + # f"incoming_probe_value should always be a string (got {type(probe_value)} / {probe_value})" + # ) + probe_value = str(probe_value) return probe_value def outgoing_probe_value(self, outgoing_probe_value): From 4a72eda49ce5750ea3a45542d073178a6bf7538c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 18 Dec 2024 20:43:39 +0000 Subject: [PATCH 235/238] adding form actions XSS detect, fixing parameter extraction bugs, deps update --- bbot/core/helpers/regexes.py | 5 +- bbot/modules/internal/excavate.py | 10 +- bbot/modules/lightfuzz_submodules/base.py | 18 +- bbot/modules/lightfuzz_submodules/xss.py | 34 +- .../module_tests/test_module_lightfuzz.py | 47 +++ poetry.lock | 375 ++++++++++++------ 6 files changed, 338 insertions(+), 151 deletions(-) diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py index 6025ae297..4e3c1e425 100644 --- a/bbot/core/helpers/regexes.py +++ b/bbot/core/helpers/regexes.py @@ -120,12 +120,13 @@ # For use with excavate parameters extractor input_tag_regex = re.compile( - r"]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([%\-\._=+\/\w]*)[\"\']?[^>]*?>" + r"]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?\svalue=[\"\']?([:%\-\._=+\/\w]*)[\"\']?[^>]*?>" ) input_tag_regex2 = re.compile( - r"]*?value=[\"\']?([\-%\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>" + r"]*?\svalue=[\"\']?([:\-%\._=+\/\w]*)[\"\']?[^>]*?\sname=[\"\']?([\-\._=+\/\w]+)[\"\']?[^>]*?>" ) input_tag_novalue_regex = re.compile(r"]*\bvalue=)[^>]*?name=[\"\']?([\-\._=+\/\w]*)[\"\']?[^>]*?>") +input_tag_novalue_regex = re.compile(r"]*\b\svalue=)[^>]*?\sname=[\"\']?([\-\._=+\/\w]*)[\"\']?[^>]*?>") # jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=") # jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}") # jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}") diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 1ba79018c..adacc0169 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -493,11 +493,11 @@ class GetForm(ParameterExtractorRule): form_content_regexes = { "input_tag_regex": bbot_regexes.input_tag_regex, "input_tag_regex2": bbot_regexes.input_tag_regex2, - "input_tag_novalue_regex": bbot_regexes.input_tag_novalue_regex, "select_tag_regex": bbot_regexes.select_tag_regex, "textarea_tag_regex": bbot_regexes.textarea_tag_regex, "button_tag_regex": bbot_regexes.button_tag_regex, "button_tag_regex2": bbot_regexes.button_tag_regex2, + "_input_tag_novalue_regex": bbot_regexes.input_tag_novalue_regex, } extraction_regex = bbot_regexes.get_form_regex output_type = "GETPARAM" @@ -515,15 +515,17 @@ def extract(self): for form_content_regex_name, form_content_regex in self.form_content_regexes.items(): input_tags = form_content_regex.findall(form_content) if input_tags: - if form_content_regex_name == "input_tag_novalue_regex": - form_parameters[input_tags[0]] = None + if form_content_regex_name == "_input_tag_novalue_regex": + form_parameters.setdefault(input_tags[0], None) else: if form_content_regex_name in ["input_tag_regex2", "button_tag_regex2"]: input_tags = [(b, a) for a, b in input_tags] for parameter_name, original_value in input_tags: - form_parameters[parameter_name] = original_value.strip() + # form_parameters[parameter_name] = original_value.strip() + form_parameters.setdefault(parameter_name, original_value.strip()) + for parameter_name, original_value in form_parameters.items(): yield ( diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py index 32d6d8033..61e61c9d2 100644 --- a/bbot/modules/lightfuzz_submodules/base.py +++ b/bbot/modules/lightfuzz_submodules/base.py @@ -18,15 +18,6 @@ def additional_params_process(self, additional_params, additional_params_populat new_additional_params[k] = v return new_additional_params - async def send_probe(self, probe): - probe = self.outgoing_probe_value(probe) - getparams = {self.event.data["name"]: probe} - url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl() - self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}") - r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10) - if r: - return r.text - def compare_baseline( self, event_type, probe, cookies, additional_params_populate_empty=False, speculative_mode="GETPARAM" ): @@ -131,6 +122,7 @@ async def compare_probe( compare_result = await http_compare.compare( self.event.data["url"], method="POST", data=data, cookies=cookies ) + elif event_type == "BODYJSON": data = {self.event.data["name"]: f"{probe}"} if additional_params: @@ -150,6 +142,11 @@ async def standard_probe( speculative_mode="GETPARAM", allow_redirects=False, ): + + + + + probe = self.outgoing_probe_value(probe) if event_type == "SPECULATIVE": @@ -177,6 +174,8 @@ async def standard_probe( if event_type == "POSTPARAM": + + method = "POST" data = {self.event.data["name"]: probe} if self.event.data["additional_params"] is not None: @@ -185,6 +184,7 @@ async def standard_probe( self.event.data["additional_params"], additional_params_populate_empty ) ) + elif event_type == "BODYJSON": method = "POST" json_data = {self.event.data["name"]: probe} diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 26cb00d28..4f157d831 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -4,7 +4,7 @@ class XSSLightfuzz(BaseLightfuzz): - def determine_context(self, html, random_string): + def determine_context(self, cookies, html, random_string): between_tags = False in_tag_attribute = False in_javascript = False @@ -29,20 +29,22 @@ def determine_context(self, html, random_string): return between_tags, in_tag_attribute, in_javascript - async def check_probe(self, probe, match, context): - probe_result = await self.send_probe(probe) - if probe_result and match in probe_result: + async def check_probe(self, cookies, probe, match, context): + probe_result = await self.standard_probe(self.event.data["type"], cookies, probe) + if probe_result and match in probe_result.text: self.results.append( { "type": "FINDING", - "description": f"Possible Reflected XSS. Parameter: [{self.event.data['name']}] Context: [{context}]", + "description": f"Possible Reflected XSS. Parameter: [{self.event.data['name']}] Context: [{context}] Parameter Type: [{self.event.data['type']}]", } ) return True return False async def fuzz(self): + lightfuzz_event = self.event.parent + cookies = self.event.data.get("assigned_cookies", {}) # If this came from paramminer_getparams and didn't have a http_reflection tag, we don't need to check again if ( @@ -57,15 +59,16 @@ async def fuzz(self): reflection = None random_string = self.lightfuzz.helpers.rand_string(8) - reflection_probe_result = await self.send_probe(random_string) - if reflection_probe_result and random_string in reflection_probe_result: + + reflection_probe_result = await self.standard_probe(self.event.data["type"], cookies, random_string) + self.lightfuzz.hugeinfo(reflection_probe_result.text) + if reflection_probe_result and random_string in reflection_probe_result.text: reflection = True if not reflection or reflection is False: return - between_tags, in_tag_attribute, in_javascript = self.determine_context(reflection_probe_result, random_string) - + between_tags, in_tag_attribute, in_javascript = self.determine_context(cookies, reflection_probe_result.text, random_string) self.lightfuzz.debug( f"determine_context returned: between_tags [{between_tags}], in_tag_attribute [{in_tag_attribute}], in_javascript [{in_javascript}]" ) @@ -73,22 +76,27 @@ async def fuzz(self): if between_tags: for tag in tags: between_tags_probe = f"<{tag}>{random_string}" - result = await self.check_probe(between_tags_probe, between_tags_probe, f"Between Tags ({tag} tag)") + result = await self.check_probe(cookies, between_tags_probe, between_tags_probe, f"Between Tags ({tag} tag)") if result is True: break if in_tag_attribute: in_tag_attribute_probe = f'{random_string}"' in_tag_attribute_match = f'"{random_string}""' - await self.check_probe(in_tag_attribute_probe, in_tag_attribute_match, "Tag Attribute") + await self.check_probe(cookies, in_tag_attribute_probe, in_tag_attribute_match, "Tag Attribute") + + + in_tag_attribute_probe = f'javascript:{random_string}' + in_tag_attribute_match = f'action="javascript:{random_string}' + await self.check_probe(cookies, in_tag_attribute_probe, in_tag_attribute_match, "Form Action Injection") if in_javascript: in_javascript_probe = rf"" - result = await self.check_probe(in_javascript_probe, in_javascript_probe, "In Javascript") + result = await self.check_probe(cookies, in_javascript_probe, in_javascript_probe, "In Javascript") if result is False: in_javasscript_escape_probe = rf"a\';zzzzz({random_string})\\" in_javasscript_escape_match = rf"a\\';zzzzz({random_string})\\" - await self.check_probe( + await self.check_probe(cookies, in_javasscript_escape_probe, in_javasscript_escape_match, "In Javascript (escaping the escape character)", diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py index 4ff3a478c..0ad4d08df 100644 --- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py +++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py @@ -225,6 +225,53 @@ def check(self, module_test, events): assert web_parameter_emitted, "WEB_PARAMETER was not emitted" assert xss_finding_emitted, "Between Tags XSS FINDING not emitted" +# Form Action Injection Detection +class Test_Lightfuzz_xss_formaction(Test_Lightfuzz_xss): + + def request_handler(self, request): + form_data = request.form + value = form_data.get("func", None) + + parameter_block = """ +
+
+ + + +
+
+ """ + + if value: + xss_block = f""" +
+
+ + + +
+
+ """ + + + return Response(xss_block, status=200) + + return Response(parameter_block, status=200) + + def check(self, module_test, events): + web_parameter_emitted = False + xss_finding_emitted = False + for e in events: + if e.type == "WEB_PARAMETER": + if "HTTP Extracted Parameter [search]" in e.data["description"]: + web_parameter_emitted = True + + if e.type == "FINDING": + if "Possible Reflected XSS. Parameter: [func] Context: [Form Action Injection] Parameter Type: [POSTPARAM]" in e.data["description"]: + xss_finding_emitted = True + + assert web_parameter_emitted, "WEB_PARAMETER was not emitted" + assert xss_finding_emitted, "Form Action XSS FINDING not emitted" # Base64 Envelope XSS Detection class Test_Lightfuzz_envelope_base64(Test_Lightfuzz_xss): diff --git a/poetry.lock b/poetry.lock index 2e54a2edf..181ddebcc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "annotated-types" version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -15,6 +16,7 @@ files = [ name = "ansible" version = "8.7.0" description = "Radically simple IT automation" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -29,6 +31,7 @@ ansible-core = ">=2.15.7,<2.16.0" name = "ansible-core" version = "2.15.13" description = "Radically simple IT automation" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -48,6 +51,7 @@ resolvelib = ">=0.5.3,<1.1.0" name = "ansible-runner" version = "2.4.0" description = "\"Consistent Ansible Python API and CLI with container and process isolation runtime capabilities\"" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -66,6 +70,7 @@ pyyaml = "*" name = "antlr4-python3-runtime" version = "4.9.3" description = "ANTLR 4.9.3 runtime for Python 3.7" +category = "main" optional = false python-versions = "*" files = [ @@ -74,30 +79,32 @@ files = [ [[package]] name = "anyio" -version = "4.6.2.post1" +version = "4.7.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" optional = false python-versions = ">=3.9" files = [ - {file = "anyio-4.6.2.post1-py3-none-any.whl", hash = "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d"}, - {file = "anyio-4.6.2.post1.tar.gz", hash = "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c"}, + {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"}, + {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"}, ] [package.dependencies] exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" -typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] -doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21.0b1)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] trio = ["trio (>=0.26.1)"] [[package]] name = "babel" version = "2.16.0" description = "Internationalization utilities" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -112,6 +119,7 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] name = "beautifulsoup4" version = "4.12.3" description = "Screen-scraping library" +category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -133,6 +141,7 @@ lxml = ["lxml"] name = "cachetools" version = "5.5.0" description = "Extensible memoizing collections and decorators" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -142,19 +151,21 @@ files = [ [[package]] name = "certifi" -version = "2024.8.30" +version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, + {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, ] [[package]] name = "cffi" version = "1.17.1" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -234,6 +245,7 @@ pycparser = "*" name = "cfgv" version = "3.4.0" description = "Validate configuration and produce human readable error messages." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -245,6 +257,7 @@ files = [ name = "charset-normalizer" version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -359,6 +372,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -373,6 +387,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "cloudcheck" version = "6.0.0.686" description = "Check whether an IP address belongs to a cloud provider" +category = "main" optional = false python-versions = "<4.0,>=3.9" files = [ @@ -390,6 +405,7 @@ regex = ">=2024.4.16,<2025.0.0" name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -399,73 +415,74 @@ files = [ [[package]] name = "coverage" -version = "7.6.8" +version = "7.6.9" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.9" files = [ - {file = "coverage-7.6.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b39e6011cd06822eb964d038d5dff5da5d98652b81f5ecd439277b32361a3a50"}, - {file = "coverage-7.6.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:63c19702db10ad79151a059d2d6336fe0c470f2e18d0d4d1a57f7f9713875dcf"}, - {file = "coverage-7.6.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3985b9be361d8fb6b2d1adc9924d01dec575a1d7453a14cccd73225cb79243ee"}, - {file = "coverage-7.6.8-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:644ec81edec0f4ad17d51c838a7d01e42811054543b76d4ba2c5d6af741ce2a6"}, - {file = "coverage-7.6.8-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f188a2402f8359cf0c4b1fe89eea40dc13b52e7b4fd4812450da9fcd210181d"}, - {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e19122296822deafce89a0c5e8685704c067ae65d45e79718c92df7b3ec3d331"}, - {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13618bed0c38acc418896005732e565b317aa9e98d855a0e9f211a7ffc2d6638"}, - {file = "coverage-7.6.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:193e3bffca48ad74b8c764fb4492dd875038a2f9925530cb094db92bb5e47bed"}, - {file = "coverage-7.6.8-cp310-cp310-win32.whl", hash = "sha256:3988665ee376abce49613701336544041f2117de7b7fbfe91b93d8ff8b151c8e"}, - {file = "coverage-7.6.8-cp310-cp310-win_amd64.whl", hash = "sha256:f56f49b2553d7dd85fd86e029515a221e5c1f8cb3d9c38b470bc38bde7b8445a"}, - {file = "coverage-7.6.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:86cffe9c6dfcfe22e28027069725c7f57f4b868a3f86e81d1c62462764dc46d4"}, - {file = "coverage-7.6.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d82ab6816c3277dc962cfcdc85b1efa0e5f50fb2c449432deaf2398a2928ab94"}, - {file = "coverage-7.6.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13690e923a3932e4fad4c0ebfb9cb5988e03d9dcb4c5150b5fcbf58fd8bddfc4"}, - {file = "coverage-7.6.8-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4be32da0c3827ac9132bb488d331cb32e8d9638dd41a0557c5569d57cf22c9c1"}, - {file = "coverage-7.6.8-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44e6c85bbdc809383b509d732b06419fb4544dca29ebe18480379633623baafb"}, - {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:768939f7c4353c0fac2f7c37897e10b1414b571fd85dd9fc49e6a87e37a2e0d8"}, - {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e44961e36cb13c495806d4cac67640ac2866cb99044e210895b506c26ee63d3a"}, - {file = "coverage-7.6.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ea8bb1ab9558374c0ab591783808511d135a833c3ca64a18ec927f20c4030f0"}, - {file = "coverage-7.6.8-cp311-cp311-win32.whl", hash = "sha256:629a1ba2115dce8bf75a5cce9f2486ae483cb89c0145795603d6554bdc83e801"}, - {file = "coverage-7.6.8-cp311-cp311-win_amd64.whl", hash = "sha256:fb9fc32399dca861584d96eccd6c980b69bbcd7c228d06fb74fe53e007aa8ef9"}, - {file = "coverage-7.6.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e683e6ecc587643f8cde8f5da6768e9d165cd31edf39ee90ed7034f9ca0eefee"}, - {file = "coverage-7.6.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1defe91d41ce1bd44b40fabf071e6a01a5aa14de4a31b986aa9dfd1b3e3e414a"}, - {file = "coverage-7.6.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7ad66e8e50225ebf4236368cc43c37f59d5e6728f15f6e258c8639fa0dd8e6d"}, - {file = "coverage-7.6.8-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fe47da3e4fda5f1abb5709c156eca207eacf8007304ce3019eb001e7a7204cb"}, - {file = "coverage-7.6.8-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:202a2d645c5a46b84992f55b0a3affe4f0ba6b4c611abec32ee88358db4bb649"}, - {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4674f0daa1823c295845b6a740d98a840d7a1c11df00d1fd62614545c1583787"}, - {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:74610105ebd6f33d7c10f8907afed696e79c59e3043c5f20eaa3a46fddf33b4c"}, - {file = "coverage-7.6.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:37cda8712145917105e07aab96388ae76e787270ec04bcb9d5cc786d7cbb8443"}, - {file = "coverage-7.6.8-cp312-cp312-win32.whl", hash = "sha256:9e89d5c8509fbd6c03d0dd1972925b22f50db0792ce06324ba069f10787429ad"}, - {file = "coverage-7.6.8-cp312-cp312-win_amd64.whl", hash = "sha256:379c111d3558272a2cae3d8e57e6b6e6f4fe652905692d54bad5ea0ca37c5ad4"}, - {file = "coverage-7.6.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b0c69f4f724c64dfbfe79f5dfb503b42fe6127b8d479b2677f2b227478db2eb"}, - {file = "coverage-7.6.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c15b32a7aca8038ed7644f854bf17b663bc38e1671b5d6f43f9a2b2bd0c46f63"}, - {file = "coverage-7.6.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63068a11171e4276f6ece913bde059e77c713b48c3a848814a6537f35afb8365"}, - {file = "coverage-7.6.8-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f4548c5ead23ad13fb7a2c8ea541357474ec13c2b736feb02e19a3085fac002"}, - {file = "coverage-7.6.8-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b4b4299dd0d2c67caaaf286d58aef5e75b125b95615dda4542561a5a566a1e3"}, - {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9ebfb2507751f7196995142f057d1324afdab56db1d9743aab7f50289abd022"}, - {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c1b4474beee02ede1eef86c25ad4600a424fe36cff01a6103cb4533c6bf0169e"}, - {file = "coverage-7.6.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9fd2547e6decdbf985d579cf3fc78e4c1d662b9b0ff7cc7862baaab71c9cc5b"}, - {file = "coverage-7.6.8-cp313-cp313-win32.whl", hash = "sha256:8aae5aea53cbfe024919715eca696b1a3201886ce83790537d1c3668459c7146"}, - {file = "coverage-7.6.8-cp313-cp313-win_amd64.whl", hash = "sha256:ae270e79f7e169ccfe23284ff5ea2d52a6f401dc01b337efb54b3783e2ce3f28"}, - {file = "coverage-7.6.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de38add67a0af869b0d79c525d3e4588ac1ffa92f39116dbe0ed9753f26eba7d"}, - {file = "coverage-7.6.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b07c25d52b1c16ce5de088046cd2432b30f9ad5e224ff17c8f496d9cb7d1d451"}, - {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62a66ff235e4c2e37ed3b6104d8b478d767ff73838d1222132a7a026aa548764"}, - {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09b9f848b28081e7b975a3626e9081574a7b9196cde26604540582da60235fdf"}, - {file = "coverage-7.6.8-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:093896e530c38c8e9c996901858ac63f3d4171268db2c9c8b373a228f459bbc5"}, - {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a7b8ac36fd688c8361cbc7bf1cb5866977ece6e0b17c34aa0df58bda4fa18a4"}, - {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:38c51297b35b3ed91670e1e4efb702b790002e3245a28c76e627478aa3c10d83"}, - {file = "coverage-7.6.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2e4e0f60cb4bd7396108823548e82fdab72d4d8a65e58e2c19bbbc2f1e2bfa4b"}, - {file = "coverage-7.6.8-cp313-cp313t-win32.whl", hash = "sha256:6535d996f6537ecb298b4e287a855f37deaf64ff007162ec0afb9ab8ba3b8b71"}, - {file = "coverage-7.6.8-cp313-cp313t-win_amd64.whl", hash = "sha256:c79c0685f142ca53256722a384540832420dff4ab15fec1863d7e5bc8691bdcc"}, - {file = "coverage-7.6.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ac47fa29d8d41059ea3df65bd3ade92f97ee4910ed638e87075b8e8ce69599e"}, - {file = "coverage-7.6.8-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:24eda3a24a38157eee639ca9afe45eefa8d2420d49468819ac5f88b10de84f4c"}, - {file = "coverage-7.6.8-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4c81ed2820b9023a9a90717020315e63b17b18c274a332e3b6437d7ff70abe0"}, - {file = "coverage-7.6.8-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd55f8fc8fa494958772a2a7302b0354ab16e0b9272b3c3d83cdb5bec5bd1779"}, - {file = "coverage-7.6.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f39e2f3530ed1626c66e7493be7a8423b023ca852aacdc91fb30162c350d2a92"}, - {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:716a78a342679cd1177bc8c2fe957e0ab91405bd43a17094324845200b2fddf4"}, - {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:177f01eeaa3aee4a5ffb0d1439c5952b53d5010f86e9d2667963e632e30082cc"}, - {file = "coverage-7.6.8-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:912e95017ff51dc3d7b6e2be158dedc889d9a5cc3382445589ce554f1a34c0ea"}, - {file = "coverage-7.6.8-cp39-cp39-win32.whl", hash = "sha256:4db3ed6a907b555e57cc2e6f14dc3a4c2458cdad8919e40b5357ab9b6db6c43e"}, - {file = "coverage-7.6.8-cp39-cp39-win_amd64.whl", hash = "sha256:428ac484592f780e8cd7b6b14eb568f7c85460c92e2a37cb0c0e5186e1a0d076"}, - {file = "coverage-7.6.8-pp39.pp310-none-any.whl", hash = "sha256:5c52a036535d12590c32c49209e79cabaad9f9ad8aa4cbd875b68c4d67a9cbce"}, - {file = "coverage-7.6.8.tar.gz", hash = "sha256:8b2b8503edb06822c86d82fa64a4a5cb0760bb8f31f26e138ec743f422f37cfc"}, + {file = "coverage-7.6.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85d9636f72e8991a1706b2b55b06c27545448baf9f6dbf51c4004609aacd7dcb"}, + {file = "coverage-7.6.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:608a7fd78c67bee8936378299a6cb9f5149bb80238c7a566fc3e6717a4e68710"}, + {file = "coverage-7.6.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96d636c77af18b5cb664ddf12dab9b15a0cfe9c0bde715da38698c8cea748bfa"}, + {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d75cded8a3cff93da9edc31446872d2997e327921d8eed86641efafd350e1df1"}, + {file = "coverage-7.6.9-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7b15f589593110ae767ce997775d645b47e5cbbf54fd322f8ebea6277466cec"}, + {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:44349150f6811b44b25574839b39ae35291f6496eb795b7366fef3bd3cf112d3"}, + {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d891c136b5b310d0e702e186d70cd16d1119ea8927347045124cb286b29297e5"}, + {file = "coverage-7.6.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:db1dab894cc139f67822a92910466531de5ea6034ddfd2b11c0d4c6257168073"}, + {file = "coverage-7.6.9-cp310-cp310-win32.whl", hash = "sha256:41ff7b0da5af71a51b53f501a3bac65fb0ec311ebed1632e58fc6107f03b9198"}, + {file = "coverage-7.6.9-cp310-cp310-win_amd64.whl", hash = "sha256:35371f8438028fdccfaf3570b31d98e8d9eda8bb1d6ab9473f5a390969e98717"}, + {file = "coverage-7.6.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:932fc826442132dde42ee52cf66d941f581c685a6313feebed358411238f60f9"}, + {file = "coverage-7.6.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:085161be5f3b30fd9b3e7b9a8c301f935c8313dcf928a07b116324abea2c1c2c"}, + {file = "coverage-7.6.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc660a77e1c2bf24ddbce969af9447a9474790160cfb23de6be4fa88e3951c7"}, + {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c69e42c892c018cd3c8d90da61d845f50a8243062b19d228189b0224150018a9"}, + {file = "coverage-7.6.9-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0824a28ec542a0be22f60c6ac36d679e0e262e5353203bea81d44ee81fe9c6d4"}, + {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4401ae5fc52ad8d26d2a5d8a7428b0f0c72431683f8e63e42e70606374c311a1"}, + {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:98caba4476a6c8d59ec1eb00c7dd862ba9beca34085642d46ed503cc2d440d4b"}, + {file = "coverage-7.6.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ee5defd1733fd6ec08b168bd4f5387d5b322f45ca9e0e6c817ea6c4cd36313e3"}, + {file = "coverage-7.6.9-cp311-cp311-win32.whl", hash = "sha256:f2d1ec60d6d256bdf298cb86b78dd715980828f50c46701abc3b0a2b3f8a0dc0"}, + {file = "coverage-7.6.9-cp311-cp311-win_amd64.whl", hash = "sha256:0d59fd927b1f04de57a2ba0137166d31c1a6dd9e764ad4af552912d70428c92b"}, + {file = "coverage-7.6.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:99e266ae0b5d15f1ca8d278a668df6f51cc4b854513daab5cae695ed7b721cf8"}, + {file = "coverage-7.6.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9901d36492009a0a9b94b20e52ebfc8453bf49bb2b27bca2c9706f8b4f5a554a"}, + {file = "coverage-7.6.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abd3e72dd5b97e3af4246cdada7738ef0e608168de952b837b8dd7e90341f015"}, + {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff74026a461eb0660366fb01c650c1d00f833a086b336bdad7ab00cc952072b3"}, + {file = "coverage-7.6.9-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65dad5a248823a4996724a88eb51d4b31587aa7aa428562dbe459c684e5787ae"}, + {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:22be16571504c9ccea919fcedb459d5ab20d41172056206eb2994e2ff06118a4"}, + {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f957943bc718b87144ecaee70762bc2bc3f1a7a53c7b861103546d3a403f0a6"}, + {file = "coverage-7.6.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ae1387db4aecb1f485fb70a6c0148c6cdaebb6038f1d40089b1fc84a5db556f"}, + {file = "coverage-7.6.9-cp312-cp312-win32.whl", hash = "sha256:1a330812d9cc7ac2182586f6d41b4d0fadf9be9049f350e0efb275c8ee8eb692"}, + {file = "coverage-7.6.9-cp312-cp312-win_amd64.whl", hash = "sha256:b12c6b18269ca471eedd41c1b6a1065b2f7827508edb9a7ed5555e9a56dcfc97"}, + {file = "coverage-7.6.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:899b8cd4781c400454f2f64f7776a5d87bbd7b3e7f7bda0cb18f857bb1334664"}, + {file = "coverage-7.6.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:61f70dc68bd36810972e55bbbe83674ea073dd1dcc121040a08cdf3416c5349c"}, + {file = "coverage-7.6.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a289d23d4c46f1a82d5db4abeb40b9b5be91731ee19a379d15790e53031c014"}, + {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e216d8044a356fc0337c7a2a0536d6de07888d7bcda76febcb8adc50bdbbd00"}, + {file = "coverage-7.6.9-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c026eb44f744acaa2bda7493dad903aa5bf5fc4f2554293a798d5606710055d"}, + {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e77363e8425325384f9d49272c54045bbed2f478e9dd698dbc65dbc37860eb0a"}, + {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:777abfab476cf83b5177b84d7486497e034eb9eaea0d746ce0c1268c71652077"}, + {file = "coverage-7.6.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:447af20e25fdbe16f26e84eb714ba21d98868705cb138252d28bc400381f6ffb"}, + {file = "coverage-7.6.9-cp313-cp313-win32.whl", hash = "sha256:d872ec5aeb086cbea771c573600d47944eea2dcba8be5f3ee649bfe3cb8dc9ba"}, + {file = "coverage-7.6.9-cp313-cp313-win_amd64.whl", hash = "sha256:fd1213c86e48dfdc5a0cc676551db467495a95a662d2396ecd58e719191446e1"}, + {file = "coverage-7.6.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9e7484d286cd5a43744e5f47b0b3fb457865baf07bafc6bee91896364e1419"}, + {file = "coverage-7.6.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1cf0872ee455c03e5674b5bca5e3e68e159379c1af0903e89f5eba9ccc3a"}, + {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d10e07aa2b91835d6abec555ec8b2733347956991901eea6ffac295f83a30e4"}, + {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13a9e2d3ee855db3dd6ea1ba5203316a1b1fd8eaeffc37c5b54987e61e4194ae"}, + {file = "coverage-7.6.9-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c38bf15a40ccf5619fa2fe8f26106c7e8e080d7760aeccb3722664c8656b030"}, + {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d5275455b3e4627c8e7154feaf7ee0743c2e7af82f6e3b561967b1cca755a0be"}, + {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8f8770dfc6e2c6a2d4569f411015c8d751c980d17a14b0530da2d7f27ffdd88e"}, + {file = "coverage-7.6.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8d2dfa71665a29b153a9681edb1c8d9c1ea50dfc2375fb4dac99ea7e21a0bcd9"}, + {file = "coverage-7.6.9-cp313-cp313t-win32.whl", hash = "sha256:5e6b86b5847a016d0fbd31ffe1001b63355ed309651851295315031ea7eb5a9b"}, + {file = "coverage-7.6.9-cp313-cp313t-win_amd64.whl", hash = "sha256:97ddc94d46088304772d21b060041c97fc16bdda13c6c7f9d8fcd8d5ae0d8611"}, + {file = "coverage-7.6.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:adb697c0bd35100dc690de83154627fbab1f4f3c0386df266dded865fc50a902"}, + {file = "coverage-7.6.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:be57b6d56e49c2739cdf776839a92330e933dd5e5d929966fbbd380c77f060be"}, + {file = "coverage-7.6.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1592791f8204ae9166de22ba7e6705fa4ebd02936c09436a1bb85aabca3e599"}, + {file = "coverage-7.6.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e12ae8cc979cf83d258acb5e1f1cf2f3f83524d1564a49d20b8bec14b637f08"}, + {file = "coverage-7.6.9-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb5555cff66c4d3d6213a296b360f9e1a8e323e74e0426b6c10ed7f4d021e464"}, + {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b9389a429e0e5142e69d5bf4a435dd688c14478a19bb901735cdf75e57b13845"}, + {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:592ac539812e9b46046620341498caf09ca21023c41c893e1eb9dbda00a70cbf"}, + {file = "coverage-7.6.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a27801adef24cc30871da98a105f77995e13a25a505a0161911f6aafbd66e678"}, + {file = "coverage-7.6.9-cp39-cp39-win32.whl", hash = "sha256:8e3c3e38930cfb729cb8137d7f055e5a473ddaf1217966aa6238c88bd9fd50e6"}, + {file = "coverage-7.6.9-cp39-cp39-win_amd64.whl", hash = "sha256:e28bf44afa2b187cc9f41749138a64435bf340adfcacb5b2290c070ce99839d4"}, + {file = "coverage-7.6.9-pp39.pp310-none-any.whl", hash = "sha256:f3ca78518bc6bc92828cd11867b121891d75cae4ea9e908d72030609b996db1b"}, + {file = "coverage-7.6.9.tar.gz", hash = "sha256:4a8d8977b0c6ef5aeadcb644da9e69ae0dcfe66ec7f368c89c72e058bd71164d"}, ] [package.dependencies] @@ -478,6 +495,7 @@ toml = ["tomli"] name = "cryptography" version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -527,6 +545,7 @@ test-randomorder = ["pytest-randomly"] name = "deepdiff" version = "7.0.1" description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -545,6 +564,7 @@ optimize = ["orjson"] name = "distlib" version = "0.3.9" description = "Distribution utilities" +category = "dev" optional = false python-versions = "*" files = [ @@ -556,6 +576,7 @@ files = [ name = "dnspython" version = "2.7.0" description = "DNS toolkit" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -576,6 +597,7 @@ wmi = ["wmi (>=1.5.1)"] name = "dunamai" version = "1.23.0" description = "Dynamic version generation" +category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -590,6 +612,7 @@ packaging = ">=20.9" name = "exceptiongroup" version = "1.2.2" description = "Backport of PEP 654 (exception groups)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -604,6 +627,7 @@ test = ["pytest (>=6)"] name = "fastapi" version = "0.115.6" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -624,6 +648,7 @@ standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "htt name = "filelock" version = "3.16.1" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -640,6 +665,7 @@ typing = ["typing-extensions (>=4.12.2)"] name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." +category = "dev" optional = false python-versions = "*" files = [ @@ -657,6 +683,7 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "griffe" version = "1.5.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -671,6 +698,7 @@ colorama = ">=0.4" name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -682,6 +710,7 @@ files = [ name = "httpcore" version = "1.0.7" description = "A minimal low-level HTTP client." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -696,13 +725,14 @@ h11 = ">=0.13,<0.15" [package.extras] asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httpx" version = "0.27.2" description = "The next generation HTTP client." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -713,26 +743,27 @@ files = [ [package.dependencies] anyio = "*" certifi = "*" -httpcore = "==1.*" +httpcore = ">=1.0.0,<2.0.0" idna = "*" sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (==1.*)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] zstd = ["zstandard (>=0.18.0)"] [[package]] name = "identify" -version = "2.6.2" +version = "2.6.3" description = "File identification library for Python" +category = "dev" optional = false python-versions = ">=3.9" files = [ - {file = "identify-2.6.2-py2.py3-none-any.whl", hash = "sha256:c097384259f49e372f4ea00a19719d95ae27dd5ff0fd77ad630aa891306b82f3"}, - {file = "identify-2.6.2.tar.gz", hash = "sha256:fab5c716c24d7a789775228823797296a2994b075fb6080ac83a102772a98cbd"}, + {file = "identify-2.6.3-py2.py3-none-any.whl", hash = "sha256:9edba65473324c2ea9684b1f944fe3191db3345e50b6d04571d10ed164f8d7bd"}, + {file = "identify-2.6.3.tar.gz", hash = "sha256:62f5dae9b5fef52c84cc188514e9ea4f3f636b1d8799ab5ebc475471f9e47a02"}, ] [package.extras] @@ -742,6 +773,7 @@ license = ["ukkonen"] name = "idna" version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -756,6 +788,7 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 name = "importlib-metadata" version = "6.2.1" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -775,6 +808,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.0.7" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -790,6 +824,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-black (>=0.3.7)", "pytest-checkdocs name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -801,6 +836,7 @@ files = [ name = "jinja2" version = "3.1.4" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -818,6 +854,7 @@ i18n = ["Babel (>=2.7)"] name = "libsass" version = "0.23.0" description = "Sass for Python: A straightforward binding of libsass for Python." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -833,6 +870,7 @@ files = [ name = "livereload" version = "2.7.0" description = "Python LiveReload is an awesome tool for web developers" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -847,6 +885,7 @@ tornado = "*" name = "lockfile" version = "0.12.2" description = "Platform-independent file locking module" +category = "main" optional = false python-versions = "*" files = [ @@ -858,6 +897,7 @@ files = [ name = "lxml" version = "5.3.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1012,6 +1052,7 @@ source = ["Cython (>=3.0.11)"] name = "markdown" version = "3.7" description = "Python implementation of John Gruber's Markdown." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1030,6 +1071,7 @@ testing = ["coverage", "pyyaml"] name = "markupsafe" version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1100,6 +1142,7 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1111,6 +1154,7 @@ files = [ name = "mike" version = "2.1.3" description = "Manage multiple versions of your MkDocs-powered documentation" +category = "dev" optional = false python-versions = "*" files = [ @@ -1136,6 +1180,7 @@ test = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] name = "mkdocs" version = "1.6.1" description = "Project documentation with Markdown." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1167,6 +1212,7 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-autorefs" version = "1.2.0" description = "Automatically link across pages in MkDocs." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1183,6 +1229,7 @@ mkdocs = ">=1.1" name = "mkdocs-extra-sass-plugin" version = "0.1.0" description = "This plugin adds stylesheets to your mkdocs site from `Sass`/`SCSS`." +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1199,6 +1246,7 @@ mkdocs = ">=1.1" name = "mkdocs-get-deps" version = "0.2.0" description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1214,13 +1262,14 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.48" +version = "9.5.49" description = "Documentation that simply works" +category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.48-py3-none-any.whl", hash = "sha256:b695c998f4b939ce748adbc0d3bff73fa886a670ece948cf27818fa115dc16f8"}, - {file = "mkdocs_material-9.5.48.tar.gz", hash = "sha256:a582531e8b34f4c7ed38c29d5c44763053832cf2a32f7409567e0c74749a47db"}, + {file = "mkdocs_material-9.5.49-py3-none-any.whl", hash = "sha256:c3c2d8176b18198435d3a3e119011922f3e11424074645c24019c2dcf08a360e"}, + {file = "mkdocs_material-9.5.49.tar.gz", hash = "sha256:3671bb282b4f53a1c72e08adbe04d2481a98f85fed392530051f80ff94a9621d"}, ] [package.dependencies] @@ -1245,6 +1294,7 @@ recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2. name = "mkdocs-material-extensions" version = "1.3.1" description = "Extension pack for Python Markdown and MkDocs Material." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1256,6 +1306,7 @@ files = [ name = "mkdocstrings" version = "0.27.0" description = "Automatic documentation from sources, for MkDocs." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1284,6 +1335,7 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] name = "mkdocstrings-python" version = "1.12.2" description = "A Python handler for mkdocstrings." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1300,6 +1352,7 @@ mkdocstrings = ">=0.26" name = "mmh3" version = "5.0.1" description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1413,6 +1466,7 @@ type = ["mypy (==1.11.2)"] name = "nodeenv" version = "1.9.1" description = "Node.js virtual environment builder" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -1424,6 +1478,7 @@ files = [ name = "omegaconf" version = "2.3.0" description = "A flexible configuration library" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1432,13 +1487,14 @@ files = [ ] [package.dependencies] -antlr4-python3-runtime = "==4.9.*" +antlr4-python3-runtime = ">=4.9.0,<4.10.0" PyYAML = ">=5.1.0" [[package]] name = "ordered-set" version = "4.1.0" description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1453,6 +1509,7 @@ dev = ["black", "mypy", "pytest"] name = "orjson" version = "3.10.12" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1537,6 +1594,7 @@ files = [ name = "packaging" version = "24.2" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1548,6 +1606,7 @@ files = [ name = "paginate" version = "0.5.7" description = "Divides large result sets into pages for easier browsing" +category = "dev" optional = false python-versions = "*" files = [ @@ -1563,6 +1622,7 @@ lint = ["black"] name = "pathspec" version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1574,6 +1634,7 @@ files = [ name = "pexpect" version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." +category = "main" optional = false python-versions = "*" files = [ @@ -1588,6 +1649,7 @@ ptyprocess = ">=0.5" name = "platformdirs" version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1604,6 +1666,7 @@ type = ["mypy (>=1.11.2)"] name = "pluggy" version = "1.5.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1619,6 +1682,7 @@ testing = ["pytest", "pytest-benchmark"] name = "poetry-dynamic-versioning" version = "1.4.1" description = "Plugin for Poetry to enable dynamic versioning based on VCS tags" +category = "dev" optional = false python-versions = "<4.0,>=3.7" files = [ @@ -1638,6 +1702,7 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"] name = "pre-commit" version = "4.0.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1656,6 +1721,7 @@ virtualenv = ">=20.10.0" name = "psutil" version = "6.1.0" description = "Cross-platform lib for process and system monitoring in Python." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1686,6 +1752,7 @@ test = ["pytest", "pytest-xdist", "setuptools"] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" +category = "main" optional = false python-versions = "*" files = [ @@ -1697,6 +1764,7 @@ files = [ name = "puremagic" version = "1.28" description = "Pure python implementation of magic file detection" +category = "main" optional = false python-versions = "*" files = [ @@ -1708,6 +1776,7 @@ files = [ name = "pyahocorasick" version = "2.1.0" description = "pyahocorasick is a fast and memory efficient library for exact or approximate multi-pattern string search. With the ``ahocorasick.Automaton`` class, you can find multiple key string occurrences at once in some input text. You can use it as a plain dict-like Trie or convert a Trie to an automaton for efficient Aho-Corasick search. And pickle to disk for easy reuse of large automatons. Implemented in C and tested on Python 3.6+. Works on Linux, macOS and Windows. BSD-3-Cause license." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1746,6 +1815,7 @@ testing = ["pytest", "setuptools", "twine", "wheel"] name = "pycparser" version = "2.22" description = "C parser in Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1757,6 +1827,7 @@ files = [ name = "pycryptodome" version = "3.21.0" description = "Cryptographic library for Python" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1798,6 +1869,7 @@ files = [ name = "pydantic" version = "2.10.3" description = "Data validation using Python type hints" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1818,6 +1890,7 @@ timezone = ["tzdata"] name = "pydantic-core" version = "2.27.1" description = "Core functionality for Pydantic validation and serialization" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1930,6 +2003,7 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" name = "pygments" version = "2.18.0" description = "Pygments is a syntax highlighting package written in Python." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1944,6 +2018,7 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyjwt" version = "2.10.1" description = "JSON Web Token implementation in Python" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1961,6 +2036,7 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pymdown-extensions" version = "10.12" description = "Extension pack for Python Markdown." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1979,6 +2055,7 @@ extra = ["pygments (>=2.12)"] name = "pyparsing" version = "3.2.0" description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1993,6 +2070,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pytest" version = "8.3.4" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2015,6 +2093,7 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments name = "pytest-asyncio" version = "0.25.0" description = "Pytest support for asyncio" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2033,6 +2112,7 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] name = "pytest-cov" version = "6.0.0" description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2051,6 +2131,7 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] name = "pytest-env" version = "1.1.5" description = "pytest plugin that allows you to add environment variables." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2069,6 +2150,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "pytest-mock (>=3.14)"] name = "pytest-httpserver" version = "1.1.0" description = "pytest-httpserver is a httpserver for pytest" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2083,6 +2165,7 @@ Werkzeug = ">=2.0.0" name = "pytest-httpx" version = "0.34.0" description = "Send responses to httpx." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2091,16 +2174,17 @@ files = [ ] [package.dependencies] -httpx = "==0.27.*" -pytest = "==8.*" +httpx = ">=0.27.0,<0.28.0" +pytest = ">=8.0.0,<9.0.0" [package.extras] -testing = ["pytest-asyncio (==0.24.*)", "pytest-cov (==5.*)"] +testing = ["pytest-asyncio (>=0.24.0,<0.25.0)", "pytest-cov (>=5.0.0,<6.0.0)"] [[package]] name = "pytest-rerunfailures" version = "15.0" description = "pytest plugin to re-run tests to eliminate flaky failures" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2116,6 +2200,7 @@ pytest = ">=7.4,<8.2.2 || >8.2.2" name = "pytest-timeout" version = "2.3.1" description = "pytest plugin to abort hanging tests" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2128,24 +2213,23 @@ pytest = ">=7.0.0" [[package]] name = "python-daemon" -version = "3.1.0" +version = "3.1.2" description = "Library to implement a well-behaved Unix daemon process." +category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "python_daemon-3.1.0-py3-none-any.whl", hash = "sha256:a66b5896f0aed5807a25c6128268eb496488b1f9c6927c487710049ba16be32a"}, - {file = "python_daemon-3.1.0.tar.gz", hash = "sha256:fdb621d7e5f46e74b4de1ad6b0fff6e69cd91b4f219de1476190ebdd0f4781df"}, + {file = "python_daemon-3.1.2-py3-none-any.whl", hash = "sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6"}, + {file = "python_daemon-3.1.2.tar.gz", hash = "sha256:f7b04335adc473de877f5117e26d5f1142f4c9f7cd765408f0877757be5afbf4"}, ] [package.dependencies] lockfile = ">=0.10" -setuptools = ">=62.4.0" [package.extras] build = ["build", "changelog-chug", "docutils", "python-daemon[doc]", "wheel"] devel = ["python-daemon[dist,test]"] dist = ["python-daemon[build]", "twine"] -doc = ["Sphinx (>=4.4,<5.0)"] static-analysis = ["isort (>=5.13,<6.0)", "pip-check", "pycodestyle (>=2.12,<3.0)", "pydocstyle (>=6.3,<7.0)", "pyupgrade (>=3.17,<4.0)"] test = ["coverage", "python-daemon[build,static-analysis]", "testscenarios (>=0.4)", "testtools"] @@ -2153,6 +2237,7 @@ test = ["coverage", "python-daemon[build,static-analysis]", "testscenarios (>=0. name = "python-dateutil" version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" +category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2167,6 +2252,7 @@ six = ">=1.5" name = "pyyaml" version = "6.0.2" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2229,6 +2315,7 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2243,6 +2330,7 @@ pyyaml = "*" name = "pyzmq" version = "26.2.0" description = "Python bindings for 0MQ" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2364,6 +2452,7 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "radixtarget" version = "2.0.0.58" description = "Check whether an IP address belongs to a cloud provider" +category = "main" optional = false python-versions = "<4.0,>=3.9" files = [ @@ -2375,6 +2464,7 @@ files = [ name = "regex" version = "2024.11.6" description = "Alternative regular expression module, to replace re." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2478,6 +2568,7 @@ files = [ name = "requests" version = "2.32.3" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2499,6 +2590,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-file" version = "2.1.0" description = "File transport adapter for Requests" +category = "main" optional = false python-versions = "*" files = [ @@ -2513,6 +2605,7 @@ requests = ">=1.0.0" name = "resolvelib" version = "1.0.1" description = "Resolve abstract dependencies into concrete ones" +category = "main" optional = false python-versions = "*" files = [ @@ -2530,6 +2623,7 @@ test = ["commentjson", "packaging", "pytest"] name = "ruff" version = "0.8.3" description = "An extremely fast Python linter and code formatter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2557,6 +2651,7 @@ files = [ name = "setproctitle" version = "1.3.4" description = "A Python module to customize the process title" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2650,41 +2745,23 @@ files = [ [package.extras] test = ["pytest"] -[[package]] -name = "setuptools" -version = "75.6.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false -python-versions = ">=3.9" -files = [ - {file = "setuptools-75.6.0-py3-none-any.whl", hash = "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d"}, - {file = "setuptools-75.6.0.tar.gz", hash = "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] -core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] - [[package]] name = "six" -version = "1.16.0" +version = "1.17.0" description = "Python 2 and 3 compatibility utilities" +category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] [[package]] name = "sniffio" version = "1.3.1" description = "Sniff out which async library your code is running under" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2696,6 +2773,7 @@ files = [ name = "socksio" version = "1.0.0" description = "Sans-I/O implementation of SOCKS4, SOCKS4A, and SOCKS5." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2707,6 +2785,7 @@ files = [ name = "soupsieve" version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2718,6 +2797,7 @@ files = [ name = "starlette" version = "0.41.3" description = "The little ASGI library that shines." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2736,6 +2816,7 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7 name = "tabulate" version = "0.8.10" description = "Pretty-print tabular data" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2750,6 +2831,7 @@ widechars = ["wcwidth"] name = "tldextract" version = "5.1.3" description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -2769,19 +2851,51 @@ testing = ["mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ru [[package]] name = "tomli" -version = "2.1.0" +version = "2.2.1" description = "A lil' TOML parser" +category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, - {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] [[package]] name = "tomlkit" version = "0.13.2" description = "Style preserving TOML library" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2793,6 +2907,7 @@ files = [ name = "tornado" version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2813,6 +2928,7 @@ files = [ name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2824,6 +2940,7 @@ files = [ name = "unidecode" version = "1.3.8" description = "ASCII transliterations of Unicode text" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2835,6 +2952,7 @@ files = [ name = "urllib3" version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2852,6 +2970,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "uvicorn" version = "0.34.0" description = "The lightning-fast ASGI server." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2871,6 +2990,7 @@ standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", name = "verspec" version = "0.1.0" description = "Flexible version handling" +category = "dev" optional = false python-versions = "*" files = [ @@ -2883,13 +3003,14 @@ test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] [[package]] name = "virtualenv" -version = "20.27.1" +version = "20.28.0" description = "Virtual Python Environment builder" +category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "virtualenv-20.27.1-py3-none-any.whl", hash = "sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4"}, - {file = "virtualenv-20.27.1.tar.gz", hash = "sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba"}, + {file = "virtualenv-20.28.0-py3-none-any.whl", hash = "sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0"}, + {file = "virtualenv-20.28.0.tar.gz", hash = "sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa"}, ] [package.dependencies] @@ -2905,6 +3026,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "watchdog" version = "6.0.0" description = "Filesystem events monitoring" +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2947,6 +3069,7 @@ watchmedo = ["PyYAML (>=3.10)"] name = "websockets" version = "12.0" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3028,6 +3151,7 @@ files = [ name = "werkzeug" version = "3.1.3" description = "The comprehensive WSGI web application library." +category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -3045,6 +3169,7 @@ watchdog = ["watchdog (>=2.3)"] name = "wordninja" version = "2.0.0" description = "Probabilistically split concatenated words using NLP based on English Wikipedia uni-gram frequencies." +category = "main" optional = false python-versions = "*" files = [ @@ -3055,6 +3180,7 @@ files = [ name = "xmltodict" version = "0.14.2" description = "Makes working with XML feel like you are working with JSON" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3066,6 +3192,7 @@ files = [ name = "xmltojson" version = "2.0.3" description = "A Python module and cli tool to quickly convert xml text or files into json" +category = "main" optional = false python-versions = "<4.0,>=3.7" files = [ @@ -3080,6 +3207,7 @@ xmltodict = "0.14.2" name = "yara-python" version = "4.5.1" description = "Python interface for YARA" +category = "main" optional = false python-versions = "*" files = [ @@ -3169,6 +3297,7 @@ files = [ name = "zipp" version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.9" files = [ @@ -3187,4 +3316,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "e47772530c41a0fad08b26e205811fd6c14628ab167f6e852899927c7c812e00" +content-hash = "5b7316c4fbfddce5a1d70a874070d8edc83320ff73fea32801fecc0acffc0d17" From e98d2e879d10d11ce5a456cf3a52197847bd154c Mon Sep 17 00:00:00 2001 From: liquidsec Date: Wed, 18 Dec 2024 20:51:29 +0000 Subject: [PATCH 236/238] remove debug msg --- bbot/modules/lightfuzz_submodules/xss.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/modules/lightfuzz_submodules/xss.py b/bbot/modules/lightfuzz_submodules/xss.py index 4f157d831..9ac606e48 100644 --- a/bbot/modules/lightfuzz_submodules/xss.py +++ b/bbot/modules/lightfuzz_submodules/xss.py @@ -61,7 +61,6 @@ async def fuzz(self): random_string = self.lightfuzz.helpers.rand_string(8) reflection_probe_result = await self.standard_probe(self.event.data["type"], cookies, random_string) - self.lightfuzz.hugeinfo(reflection_probe_result.text) if reflection_probe_result and random_string in reflection_probe_result.text: reflection = True From 8808fd3473a17cc4036938d20bbb95ebf5b14855 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Thu, 19 Dec 2024 11:58:45 -0500 Subject: [PATCH 237/238] better handling of malformed url errors during parameter extraction --- bbot/modules/internal/excavate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index adacc0169..7db6db2f9 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -612,6 +612,9 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte if self.excavate.helpers.validate_parameter(parameter_name, parameter_type): if self.excavate.in_bl(parameter_name) is False: parsed_url = urlparse(url) + if not parsed_url.hostname: + self.excavate.warning(f"Error Parsing reconstructed URL [{url}] during parameter extraction, missing hostname") + continue description = f"HTTP Extracted Parameter [{parameter_name}] ({parameterExtractorSubModule.name} Submodule)" data = { "host": parsed_url.hostname, From 85bd5ecb64feb3cc7b644d3cd8f6306fd77d6be7 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 20 Dec 2024 10:57:57 -0500 Subject: [PATCH 238/238] update poetry.lock --- poetry.lock | 218 ++++++++++++---------------------------------------- 1 file changed, 48 insertions(+), 170 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6f546adf7..6a5f69cc9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,9 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "annotated-types" version = "0.7.0" description = "Reusable constraint types to use with typing.Annotated" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -16,7 +15,6 @@ files = [ name = "ansible" version = "8.7.0" description = "Radically simple IT automation" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -31,7 +29,6 @@ ansible-core = ">=2.15.7,<2.16.0" name = "ansible-core" version = "2.15.13" description = "Radically simple IT automation" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -51,7 +48,6 @@ resolvelib = ">=0.5.3,<1.1.0" name = "ansible-runner" version = "2.4.0" description = "\"Consistent Ansible Python API and CLI with container and process isolation runtime capabilities\"" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -70,7 +66,6 @@ pyyaml = "*" name = "antlr4-python3-runtime" version = "4.9.3" description = "ANTLR 4.9.3 runtime for Python 3.7" -category = "main" optional = false python-versions = "*" files = [ @@ -81,7 +76,6 @@ files = [ name = "anyio" version = "4.7.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -104,7 +98,6 @@ trio = ["trio (>=0.26.1)"] name = "babel" version = "2.16.0" description = "Internationalization utilities" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -119,7 +112,6 @@ dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] name = "beautifulsoup4" version = "4.12.3" description = "Screen-scraping library" -category = "main" optional = false python-versions = ">=3.6.0" files = [ @@ -141,7 +133,6 @@ lxml = ["lxml"] name = "cachetools" version = "5.5.0" description = "Extensible memoizing collections and decorators" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -153,7 +144,6 @@ files = [ name = "certifi" version = "2024.12.14" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -165,7 +155,6 @@ files = [ name = "cffi" version = "1.17.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -245,7 +234,6 @@ pycparser = "*" name = "cfgv" version = "3.4.0" description = "Validate configuration and produce human readable error messages." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -257,7 +245,6 @@ files = [ name = "charset-normalizer" version = "3.4.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -372,7 +359,6 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -387,7 +373,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "cloudcheck" version = "7.0.33" description = "Check whether an IP address belongs to a cloud provider" -category = "main" optional = false python-versions = "<4.0,>=3.9" files = [ @@ -405,7 +390,6 @@ regex = ">=2024.4.16,<2025.0.0" name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -417,7 +401,6 @@ files = [ name = "coverage" version = "7.6.9" description = "Code coverage measurement for Python" -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -495,7 +478,6 @@ toml = ["tomli"] name = "cryptography" version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -545,7 +527,6 @@ test-randomorder = ["pytest-randomly"] name = "deepdiff" version = "7.0.1" description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -564,7 +545,6 @@ optimize = ["orjson"] name = "distlib" version = "0.3.9" description = "Distribution utilities" -category = "dev" optional = false python-versions = "*" files = [ @@ -576,7 +556,6 @@ files = [ name = "dnspython" version = "2.7.0" description = "DNS toolkit" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -597,7 +576,6 @@ wmi = ["wmi (>=1.5.1)"] name = "dunamai" version = "1.23.0" description = "Dynamic version generation" -category = "dev" optional = false python-versions = ">=3.5" files = [ @@ -612,7 +590,6 @@ packaging = ">=20.9" name = "exceptiongroup" version = "1.2.2" description = "Backport of PEP 654 (exception groups)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -627,7 +604,6 @@ test = ["pytest (>=6)"] name = "fastapi" version = "0.115.6" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -648,7 +624,6 @@ standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.5)", "htt name = "filelock" version = "3.16.1" description = "A platform independent file lock." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -665,7 +640,6 @@ typing = ["typing-extensions (>=4.12.2)"] name = "ghp-import" version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." -category = "dev" optional = false python-versions = "*" files = [ @@ -683,7 +657,6 @@ dev = ["flake8", "markdown", "twine", "wheel"] name = "griffe" version = "1.5.1" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -698,7 +671,6 @@ colorama = ">=0.4" name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -710,7 +682,6 @@ files = [ name = "httpcore" version = "1.0.7" description = "A minimal low-level HTTP client." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -725,14 +696,13 @@ h11 = ">=0.13,<0.15" [package.extras] asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httpx" version = "0.27.2" description = "The next generation HTTP client." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -743,22 +713,21 @@ files = [ [package.dependencies] anyio = "*" certifi = "*" -httpcore = ">=1.0.0,<2.0.0" +httpcore = "==1.*" idna = "*" sniffio = "*" [package.extras] brotli = ["brotli", "brotlicffi"] -cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<14)"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] -socks = ["socksio (>=1.0.0,<2.0.0)"] +socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] [[package]] name = "identify" version = "2.6.3" description = "File identification library for Python" -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -773,7 +742,6 @@ license = ["ukkonen"] name = "idna" version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -788,7 +756,6 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 name = "importlib-metadata" version = "6.2.1" description = "Read metadata from Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -808,7 +775,6 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.0.7" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -824,7 +790,6 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-black (>=0.3.7)", "pytest-checkdocs name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -836,7 +801,6 @@ files = [ name = "jinja2" version = "3.1.4" description = "A very fast and expressive template engine." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -854,7 +818,6 @@ i18n = ["Babel (>=2.7)"] name = "libsass" version = "0.23.0" description = "Sass for Python: A straightforward binding of libsass for Python." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -870,7 +833,6 @@ files = [ name = "livereload" version = "2.7.1" description = "Python LiveReload is an awesome tool for web developers" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -885,7 +847,6 @@ tornado = "*" name = "lockfile" version = "0.12.2" description = "Platform-independent file locking module" -category = "main" optional = false python-versions = "*" files = [ @@ -897,7 +858,6 @@ files = [ name = "lxml" version = "5.3.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1052,7 +1012,6 @@ source = ["Cython (>=3.0.11)"] name = "markdown" version = "3.7" description = "Python implementation of John Gruber's Markdown." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1071,7 +1030,6 @@ testing = ["coverage", "pyyaml"] name = "markupsafe" version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -1142,7 +1100,6 @@ files = [ name = "mergedeep" version = "1.3.4" description = "A deep merge function for 🐍." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1154,7 +1111,6 @@ files = [ name = "mike" version = "2.1.3" description = "Manage multiple versions of your MkDocs-powered documentation" -category = "dev" optional = false python-versions = "*" files = [ @@ -1180,7 +1136,6 @@ test = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] name = "mkdocs" version = "1.6.1" description = "Project documentation with Markdown." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1212,7 +1167,6 @@ min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-imp name = "mkdocs-autorefs" version = "1.2.0" description = "Automatically link across pages in MkDocs." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1229,7 +1183,6 @@ mkdocs = ">=1.1" name = "mkdocs-extra-sass-plugin" version = "0.1.0" description = "This plugin adds stylesheets to your mkdocs site from `Sass`/`SCSS`." -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1246,7 +1199,6 @@ mkdocs = ">=1.1" name = "mkdocs-get-deps" version = "0.2.0" description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1264,7 +1216,6 @@ pyyaml = ">=5.1" name = "mkdocs-material" version = "9.5.49" description = "Documentation that simply works" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1294,7 +1245,6 @@ recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2. name = "mkdocs-material-extensions" version = "1.3.1" description = "Extension pack for Python Markdown and MkDocs Material." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1306,7 +1256,6 @@ files = [ name = "mkdocstrings" version = "0.27.0" description = "Automatic documentation from sources, for MkDocs." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1335,7 +1284,6 @@ python-legacy = ["mkdocstrings-python-legacy (>=0.2.1)"] name = "mkdocstrings-python" version = "1.12.2" description = "A Python handler for mkdocstrings." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1352,7 +1300,6 @@ mkdocstrings = ">=0.26" name = "mmh3" version = "5.0.1" description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1466,7 +1413,6 @@ type = ["mypy (==1.11.2)"] name = "nodeenv" version = "1.9.1" description = "Node.js virtual environment builder" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -1478,7 +1424,6 @@ files = [ name = "omegaconf" version = "2.3.0" description = "A flexible configuration library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1487,14 +1432,13 @@ files = [ ] [package.dependencies] -antlr4-python3-runtime = ">=4.9.0,<4.10.0" +antlr4-python3-runtime = "==4.9.*" PyYAML = ">=5.1.0" [[package]] name = "ordered-set" version = "4.1.0" description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1509,7 +1453,6 @@ dev = ["black", "mypy", "pytest"] name = "orjson" version = "3.10.12" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1594,7 +1537,6 @@ files = [ name = "packaging" version = "24.2" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1606,7 +1548,6 @@ files = [ name = "paginate" version = "0.5.7" description = "Divides large result sets into pages for easier browsing" -category = "dev" optional = false python-versions = "*" files = [ @@ -1622,7 +1563,6 @@ lint = ["black"] name = "pathspec" version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1634,7 +1574,6 @@ files = [ name = "pexpect" version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." -category = "main" optional = false python-versions = "*" files = [ @@ -1649,7 +1588,6 @@ ptyprocess = ">=0.5" name = "platformdirs" version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1666,7 +1604,6 @@ type = ["mypy (>=1.11.2)"] name = "pluggy" version = "1.5.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1682,7 +1619,6 @@ testing = ["pytest", "pytest-benchmark"] name = "poetry-dynamic-versioning" version = "1.4.1" description = "Plugin for Poetry to enable dynamic versioning based on VCS tags" -category = "dev" optional = false python-versions = "<4.0,>=3.7" files = [ @@ -1702,7 +1638,6 @@ plugin = ["poetry (>=1.2.0,<2.0.0)"] name = "pre-commit" version = "4.0.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -1719,40 +1654,38 @@ virtualenv = ">=20.10.0" [[package]] name = "psutil" -version = "6.1.0" +version = "6.1.1" description = "Cross-platform lib for process and system monitoring in Python." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"}, - {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c0e0c00aa18ca2d3b2b991643b799a15fc8f0563d2ebb6040f64ce8dc027b942"}, - {file = "psutil-6.1.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:000d1d1ebd634b4efb383f4034437384e44a6d455260aaee2eca1e9c1b55f047"}, - {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:5cd2bcdc75b452ba2e10f0e8ecc0b57b827dd5d7aaffbc6821b2a9a242823a76"}, - {file = "psutil-6.1.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:045f00a43c737f960d273a83973b2511430d61f283a44c96bf13a6e829ba8fdc"}, - {file = "psutil-6.1.0-cp27-none-win32.whl", hash = "sha256:9118f27452b70bb1d9ab3198c1f626c2499384935aaf55388211ad982611407e"}, - {file = "psutil-6.1.0-cp27-none-win_amd64.whl", hash = "sha256:a8506f6119cff7015678e2bce904a4da21025cc70ad283a53b099e7620061d85"}, - {file = "psutil-6.1.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6e2dcd475ce8b80522e51d923d10c7871e45f20918e027ab682f94f1c6351688"}, - {file = "psutil-6.1.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0895b8414afafc526712c498bd9de2b063deaac4021a3b3c34566283464aff8e"}, - {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9dcbfce5d89f1d1f2546a2090f4fcf87c7f669d1d90aacb7d7582addece9fb38"}, - {file = "psutil-6.1.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:498c6979f9c6637ebc3a73b3f87f9eb1ec24e1ce53a7c5173b8508981614a90b"}, - {file = "psutil-6.1.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d905186d647b16755a800e7263d43df08b790d709d575105d419f8b6ef65423a"}, - {file = "psutil-6.1.0-cp36-cp36m-win32.whl", hash = "sha256:6d3fbbc8d23fcdcb500d2c9f94e07b1342df8ed71b948a2649b5cb060a7c94ca"}, - {file = "psutil-6.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1209036fbd0421afde505a4879dee3b2fd7b1e14fee81c0069807adcbbcca747"}, - {file = "psutil-6.1.0-cp37-abi3-win32.whl", hash = "sha256:1ad45a1f5d0b608253b11508f80940985d1d0c8f6111b5cb637533a0e6ddc13e"}, - {file = "psutil-6.1.0-cp37-abi3-win_amd64.whl", hash = "sha256:a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"}, - {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, + {file = "psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"}, + {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"}, + {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"}, + {file = "psutil-6.1.1-cp27-none-win32.whl", hash = "sha256:6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"}, + {file = "psutil-6.1.1-cp27-none-win_amd64.whl", hash = "sha256:c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"}, + {file = "psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"}, + {file = "psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"}, + {file = "psutil-6.1.1-cp36-cp36m-win32.whl", hash = "sha256:384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"}, + {file = "psutil-6.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"}, + {file = "psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"}, + {file = "psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"}, + {file = "psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"}, ] [package.extras] -dev = ["black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "wheel"] +dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] test = ["pytest", "pytest-xdist", "setuptools"] [[package]] name = "ptyprocess" version = "0.7.0" description = "Run a subprocess in a pseudo terminal" -category = "main" optional = false python-versions = "*" files = [ @@ -1764,7 +1697,6 @@ files = [ name = "puremagic" version = "1.28" description = "Pure python implementation of magic file detection" -category = "main" optional = false python-versions = "*" files = [ @@ -1776,7 +1708,6 @@ files = [ name = "pyahocorasick" version = "2.1.0" description = "pyahocorasick is a fast and memory efficient library for exact or approximate multi-pattern string search. With the ``ahocorasick.Automaton`` class, you can find multiple key string occurrences at once in some input text. You can use it as a plain dict-like Trie or convert a Trie to an automaton for efficient Aho-Corasick search. And pickle to disk for easy reuse of large automatons. Implemented in C and tested on Python 3.6+. Works on Linux, macOS and Windows. BSD-3-Cause license." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1815,7 +1746,6 @@ testing = ["pytest", "setuptools", "twine", "wheel"] name = "pycparser" version = "2.22" description = "C parser in Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1827,7 +1757,6 @@ files = [ name = "pycryptodome" version = "3.21.0" description = "Cryptographic library for Python" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1869,7 +1798,6 @@ files = [ name = "pydantic" version = "2.10.4" description = "Data validation using Python type hints" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1890,7 +1818,6 @@ timezone = ["tzdata"] name = "pydantic-core" version = "2.27.2" description = "Core functionality for Pydantic validation and serialization" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2003,7 +1930,6 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" name = "pygments" version = "2.18.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2018,7 +1944,6 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyjwt" version = "2.10.1" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -2036,7 +1961,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pymdown-extensions" version = "10.12" description = "Extension pack for Python Markdown." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2055,7 +1979,6 @@ extra = ["pygments (>=2.12)"] name = "pyparsing" version = "3.2.0" description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2070,7 +1993,6 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pytest" version = "8.3.4" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2093,7 +2015,6 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments name = "pytest-asyncio" version = "0.25.0" description = "Pytest support for asyncio" -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2112,7 +2033,6 @@ testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] name = "pytest-cov" version = "6.0.0" description = "Pytest plugin for measuring coverage." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2131,7 +2051,6 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] name = "pytest-env" version = "1.1.5" description = "pytest plugin that allows you to add environment variables." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2150,7 +2069,6 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "pytest-mock (>=3.14)"] name = "pytest-httpserver" version = "1.1.0" description = "pytest-httpserver is a httpserver for pytest" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2165,7 +2083,6 @@ Werkzeug = ">=2.0.0" name = "pytest-httpx" version = "0.34.0" description = "Send responses to httpx." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2174,17 +2091,16 @@ files = [ ] [package.dependencies] -httpx = ">=0.27.0,<0.28.0" -pytest = ">=8.0.0,<9.0.0" +httpx = "==0.27.*" +pytest = "==8.*" [package.extras] -testing = ["pytest-asyncio (>=0.24.0,<0.25.0)", "pytest-cov (>=5.0.0,<6.0.0)"] +testing = ["pytest-asyncio (==0.24.*)", "pytest-cov (==5.*)"] [[package]] name = "pytest-rerunfailures" version = "15.0" description = "pytest plugin to re-run tests to eliminate flaky failures" -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2200,7 +2116,6 @@ pytest = ">=7.4,<8.2.2 || >8.2.2" name = "pytest-timeout" version = "2.3.1" description = "pytest plugin to abort hanging tests" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2215,7 +2130,6 @@ pytest = ">=7.0.0" name = "python-daemon" version = "3.1.2" description = "Library to implement a well-behaved Unix daemon process." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2237,7 +2151,6 @@ test = ["coverage", "python-daemon[build,static-analysis]", "testscenarios (>=0. name = "python-dateutil" version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2252,7 +2165,6 @@ six = ">=1.5" name = "pyyaml" version = "6.0.2" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2315,7 +2227,6 @@ files = [ name = "pyyaml-env-tag" version = "0.1" description = "A custom YAML tag for referencing environment variables in YAML files. " -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -2330,7 +2241,6 @@ pyyaml = "*" name = "pyzmq" version = "26.2.0" description = "Python bindings for 0MQ" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2452,7 +2362,6 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} name = "radixtarget" version = "3.0.15" description = "Check whether an IP address belongs to a cloud provider" -category = "main" optional = false python-versions = "<4.0,>=3.9" files = [ @@ -2464,7 +2373,6 @@ files = [ name = "regex" version = "2024.11.6" description = "Alternative regular expression module, to replace re." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2568,7 +2476,6 @@ files = [ name = "requests" version = "2.32.3" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2590,7 +2497,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-file" version = "2.1.0" description = "File transport adapter for Requests" -category = "main" optional = false python-versions = "*" files = [ @@ -2605,7 +2511,6 @@ requests = ">=1.0.0" name = "resolvelib" version = "1.0.1" description = "Resolve abstract dependencies into concrete ones" -category = "main" optional = false python-versions = "*" files = [ @@ -2621,37 +2526,35 @@ test = ["commentjson", "packaging", "pytest"] [[package]] name = "ruff" -version = "0.8.3" +version = "0.8.4" description = "An extremely fast Python linter and code formatter, written in Rust." -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.8.3-py3-none-linux_armv6l.whl", hash = "sha256:8d5d273ffffff0acd3db5bf626d4b131aa5a5ada1276126231c4174543ce20d6"}, - {file = "ruff-0.8.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e4d66a21de39f15c9757d00c50c8cdd20ac84f55684ca56def7891a025d7e939"}, - {file = "ruff-0.8.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c356e770811858bd20832af696ff6c7e884701115094f427b64b25093d6d932d"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c0a60a825e3e177116c84009d5ebaa90cf40dfab56e1358d1df4e29a9a14b13"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:75fb782f4db39501210ac093c79c3de581d306624575eddd7e4e13747e61ba18"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f26bc76a133ecb09a38b7868737eded6941b70a6d34ef53a4027e83913b6502"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:01b14b2f72a37390c1b13477c1c02d53184f728be2f3ffc3ace5b44e9e87b90d"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53babd6e63e31f4e96ec95ea0d962298f9f0d9cc5990a1bbb023a6baf2503a82"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ae441ce4cf925b7f363d33cd6570c51435972d697e3e58928973994e56e1452"}, - {file = "ruff-0.8.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7c65bc0cadce32255e93c57d57ecc2cca23149edd52714c0c5d6fa11ec328cd"}, - {file = "ruff-0.8.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5be450bb18f23f0edc5a4e5585c17a56ba88920d598f04a06bd9fd76d324cb20"}, - {file = "ruff-0.8.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8faeae3827eaa77f5721f09b9472a18c749139c891dbc17f45e72d8f2ca1f8fc"}, - {file = "ruff-0.8.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:db503486e1cf074b9808403991663e4277f5c664d3fe237ee0d994d1305bb060"}, - {file = "ruff-0.8.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6567be9fb62fbd7a099209257fef4ad2c3153b60579818b31a23c886ed4147ea"}, - {file = "ruff-0.8.3-py3-none-win32.whl", hash = "sha256:19048f2f878f3ee4583fc6cb23fb636e48c2635e30fb2022b3a1cd293402f964"}, - {file = "ruff-0.8.3-py3-none-win_amd64.whl", hash = "sha256:f7df94f57d7418fa7c3ffb650757e0c2b96cf2501a0b192c18e4fb5571dfada9"}, - {file = "ruff-0.8.3-py3-none-win_arm64.whl", hash = "sha256:fe2756edf68ea79707c8d68b78ca9a58ed9af22e430430491ee03e718b5e4936"}, - {file = "ruff-0.8.3.tar.gz", hash = "sha256:5e7558304353b84279042fc584a4f4cb8a07ae79b2bf3da1a7551d960b5626d3"}, + {file = "ruff-0.8.4-py3-none-linux_armv6l.whl", hash = "sha256:58072f0c06080276804c6a4e21a9045a706584a958e644353603d36ca1eb8a60"}, + {file = "ruff-0.8.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ffb60904651c00a1e0b8df594591770018a0f04587f7deeb3838344fe3adabac"}, + {file = "ruff-0.8.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ddf5d654ac0d44389f6bf05cee4caeefc3132a64b58ea46738111d687352296"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e248b1f0fa2749edd3350a2a342b67b43a2627434c059a063418e3d375cfe643"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf197b98ed86e417412ee3b6c893f44c8864f816451441483253d5ff22c0e81e"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c41319b85faa3aadd4d30cb1cffdd9ac6b89704ff79f7664b853785b48eccdf3"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9f8402b7c4f96463f135e936d9ab77b65711fcd5d72e5d67597b543bbb43cf3f"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4e56b3baa9c23d324ead112a4fdf20db9a3f8f29eeabff1355114dd96014604"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:736272574e97157f7edbbb43b1d046125fce9e7d8d583d5d65d0c9bf2c15addf"}, + {file = "ruff-0.8.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e5fe710ab6061592521f902fca7ebcb9fabd27bc7c57c764298b1c1f15fff720"}, + {file = "ruff-0.8.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:13e9ec6d6b55f6da412d59953d65d66e760d583dd3c1c72bf1f26435b5bfdbae"}, + {file = "ruff-0.8.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:97d9aefef725348ad77d6db98b726cfdb075a40b936c7984088804dfd38268a7"}, + {file = "ruff-0.8.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ab78e33325a6f5374e04c2ab924a3367d69a0da36f8c9cb6b894a62017506111"}, + {file = "ruff-0.8.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8ef06f66f4a05c3ddbc9121a8b0cecccd92c5bf3dd43b5472ffe40b8ca10f0f8"}, + {file = "ruff-0.8.4-py3-none-win32.whl", hash = "sha256:552fb6d861320958ca5e15f28b20a3d071aa83b93caee33a87b471f99a6c0835"}, + {file = "ruff-0.8.4-py3-none-win_amd64.whl", hash = "sha256:f21a1143776f8656d7f364bd264a9d60f01b7f52243fbe90e7670c0dfe0cf65d"}, + {file = "ruff-0.8.4-py3-none-win_arm64.whl", hash = "sha256:9183dd615d8df50defa8b1d9a074053891ba39025cf5ae88e8bcb52edcc4bf08"}, + {file = "ruff-0.8.4.tar.gz", hash = "sha256:0d5f89f254836799af1615798caa5f80b7f935d7a670fad66c5007928e57ace8"}, ] [[package]] name = "setproctitle" version = "1.3.4" description = "A Python module to customize the process title" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2749,7 +2652,6 @@ test = ["pytest"] name = "six" version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -2761,7 +2663,6 @@ files = [ name = "sniffio" version = "1.3.1" description = "Sniff out which async library your code is running under" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2773,7 +2674,6 @@ files = [ name = "socksio" version = "1.0.0" description = "Sans-I/O implementation of SOCKS4, SOCKS4A, and SOCKS5." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2785,7 +2685,6 @@ files = [ name = "soupsieve" version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2797,7 +2696,6 @@ files = [ name = "starlette" version = "0.41.3" description = "The little ASGI library that shines." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2816,7 +2714,6 @@ full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.7 name = "tabulate" version = "0.8.10" description = "Pretty-print tabular data" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -2831,7 +2728,6 @@ widechars = ["wcwidth"] name = "tldextract" version = "5.1.3" description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -2853,7 +2749,6 @@ testing = ["mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ru name = "tomli" version = "2.2.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2895,7 +2790,6 @@ files = [ name = "tomlkit" version = "0.13.2" description = "Style preserving TOML library" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2907,7 +2801,6 @@ files = [ name = "tornado" version = "6.4.2" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -2928,7 +2821,6 @@ files = [ name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2940,7 +2832,6 @@ files = [ name = "unidecode" version = "1.3.8" description = "ASCII transliterations of Unicode text" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2952,7 +2843,6 @@ files = [ name = "urllib3" version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2970,7 +2860,6 @@ zstd = ["zstandard (>=0.18.0)"] name = "uvicorn" version = "0.34.0" description = "The lightning-fast ASGI server." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -2990,7 +2879,6 @@ standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", name = "verspec" version = "0.1.0" description = "Flexible version handling" -category = "dev" optional = false python-versions = "*" files = [ @@ -3005,7 +2893,6 @@ test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] name = "virtualenv" version = "20.28.0" description = "Virtual Python Environment builder" -category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -3026,7 +2913,6 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "watchdog" version = "6.0.0" description = "Filesystem events monitoring" -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -3069,7 +2955,6 @@ watchmedo = ["PyYAML (>=3.10)"] name = "websockets" version = "12.0" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -3151,7 +3036,6 @@ files = [ name = "werkzeug" version = "3.1.3" description = "The comprehensive WSGI web application library." -category = "dev" optional = false python-versions = ">=3.9" files = [ @@ -3169,7 +3053,6 @@ watchdog = ["watchdog (>=2.3)"] name = "wordninja" version = "2.0.0" description = "Probabilistically split concatenated words using NLP based on English Wikipedia uni-gram frequencies." -category = "main" optional = false python-versions = "*" files = [ @@ -3180,7 +3063,6 @@ files = [ name = "xmltodict" version = "0.14.2" description = "Makes working with XML feel like you are working with JSON" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3192,7 +3074,6 @@ files = [ name = "xmltojson" version = "2.0.3" description = "A Python module and cli tool to quickly convert xml text or files into json" -category = "main" optional = false python-versions = "<4.0,>=3.7" files = [ @@ -3207,7 +3088,6 @@ xmltodict = "0.14.2" name = "yara-python" version = "4.5.1" description = "Python interface for YARA" -category = "main" optional = false python-versions = "*" files = [ @@ -3297,7 +3177,6 @@ files = [ name = "zipp" version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.9" files = [ @@ -3316,5 +3195,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "5b7316c4fbfddce5a1d70a874070d8edc83320ff73fea32801fecc0acffc0d17" - +content-hash = "eb13498c96f67946274d90932ef9a9a4cb6667a7ce5b4dc4f169b356ccc5a216"