From 5d8e0d379262bd93c2c59d04db5185c4e6df8de8 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 19:16:04 -0400
Subject: [PATCH 01/12] lightfuzz envelope system

---
 bbot/core/event/base.py                       | 407 +++++++++++++++++-
 bbot/core/helpers/misc.py                     |   2 +
 bbot/core/helpers/regexes.py                  |   6 +-
 bbot/modules/internal/excavate.py             |  44 +-
 bbot/modules/lightfuzz.py                     |  10 +
 bbot/modules/lightfuzz_submodules/base.py     |  53 ++-
 bbot/modules/lightfuzz_submodules/cmdi.py     |   3 +-
 bbot/modules/lightfuzz_submodules/crypto.py   |  31 +-
 bbot/modules/lightfuzz_submodules/path.py     |   2 +-
 bbot/modules/lightfuzz_submodules/serial.py   |   3 +-
 bbot/modules/lightfuzz_submodules/sqli.py     |   9 +-
 bbot/presets/web/lightfuzz-intense.yml        |   2 +-
 bbot/presets/web/lightfuzz-max.yml            |   2 +-
 bbot/presets/web/lightfuzz-xss.yml            |   2 +-
 bbot/presets/web/lightfuzz.yml                |   2 +-
 .../module_tests/test_module_excavate.py      |  23 +
 .../module_tests/test_module_lightfuzz.py     | 317 ++++++++++++++
 17 files changed, 860 insertions(+), 58 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index d5a9552e20..b096aa0957 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1,20 +1,26 @@
+from collections import defaultdict
 import io
 import re
+import threading
 import uuid
 import json
 import base64
+import inspect
+import asyncio
 import logging
 import tarfile
+import binascii
 import datetime
 import ipaddress
 import traceback
+import xml.etree.ElementTree as ET
 
-from copy import copy
+from copy import copy, deepcopy
 from pathlib import Path
 from typing import Optional
 from contextlib import suppress
 from radixtarget import RadixTarget
-from urllib.parse import urljoin, parse_qs
+from urllib.parse import urljoin, parse_qs, unquote, quote
 from pydantic import BaseModel, field_validator
 
 
@@ -618,6 +624,23 @@ def get_parents(self, omit=False, include_self=False):
             e = parent
         return parents
 
+    def clone(self):
+
+        # Create a shallow copy of the event first
+        cloned_event = copy(self)
+
+        # Handle attributes that need deep copying manually
+        setattr(cloned_event, "envelopes", deepcopy(self.envelopes))
+        #   cloned_event.scan = deepcopy(self.scan)
+
+        # Re-assign a new UUID
+        cloned_event.uuid = uuid.uuid4()
+
+        # Re-create the unpickleable lock object
+        cloned_event.lock = threading.RLock()
+
+        return cloned_event
+
     def _host(self):
         return ""
 
@@ -797,7 +820,13 @@ def json(self, mode="json", siem_friendly=False):
         j["discovery_path"] = self.discovery_path
         j["parent_chain"] = self.parent_chain
 
+        # parameter envelopes
+        parameter_envelopes = getattr(self, "envelopes", None)
+        if parameter_envelopes is not None:
+            j["envelopes"] = parameter_envelopes.to_dict()
+
         # normalize non-primitive python objects
+
         for k, v in list(j.items()):
             if k == "data":
                 continue
@@ -1281,12 +1310,382 @@ class URL_HINT(URL_UNVERIFIED):
 
 class WEB_PARAMETER(DictHostEvent):
 
+    class ParameterEnvelopes:
+
+        @staticmethod
+        def preprocess_base64(base64_str):
+            return base64.b64decode(base64_str).decode()
+
+        @staticmethod
+        def postprocess_base64(string):
+            return base64.b64encode(string.encode()).decode()
+
+        @staticmethod
+        def preprocess_hex(hex_str):
+            return bytes.fromhex(hex_str).decode()
+
+        @staticmethod
+        def postprocess_hex(string):
+            return string.encode().hex()
+
+        @staticmethod
+        def preprocess_urlencoded(url_encoded_str):
+            return unquote(url_encoded_str)
+
+        @staticmethod
+        def postprocess_urlencoded(string):
+            return quote(string)
+
+        @staticmethod
+        def is_ascii_printable(s):
+            return all(32 <= ord(char) < 127 for char in s)
+
+        # Converts XML ElementTree to a JSON-like dictionary
+        def xml_to_dict(self, elem):
+            """
+            Convert XML ElementTree to a dictionary recursively.
+            """
+            d = {elem.tag: {} if elem.attrib else None}
+            children = list(elem)
+            if children:
+                dd = defaultdict(list)
+                for dc in map(self.xml_to_dict, children):
+                    for k, v in dc.items():
+                        dd[k].append(v)
+                d = {elem.tag: {k: v[0] if len(v) == 1 else v for k, v in dd.items()}}
+            if elem.attrib:
+                d[elem.tag].update(("@" + k, v) for k, v in elem.attrib.items())
+            if elem.text:
+                text = elem.text.strip()
+                if children or elem.attrib:
+                    if text:
+                        d[elem.tag]["#text"] = text
+                else:
+                    d[elem.tag] = text
+            return d
+
+        def dict_to_xml(self, d):
+            """
+            Converts a dictionary to an XML string without adding an extra root node.
+            Assumes the dictionary was originally an XML structure.
+            """
+            if not isinstance(d, dict) or len(d) != 1:
+                raise ValueError("Expected a dictionary with a single root element.")
+
+            # Get the root element directly from the dict keys
+            root_tag = list(d.keys())[0]
+            root_element = ET.Element(root_tag)
+
+            # Recursive function to handle nested dicts
+            def _build_tree(element, subdict):
+                for key, value in subdict.items():
+                    if isinstance(value, dict):
+                        # Nested element
+                        sub_element = ET.SubElement(element, key)
+                        _build_tree(sub_element, value)
+                    else:
+                        # Leaf element
+                        sub_element = ET.SubElement(element, key)
+                        sub_element.text = str(value)
+
+            # Start building the tree
+            _build_tree(root_element, d[root_tag])
+
+            return ET.tostring(root_element, encoding="utf-8").decode("utf-8")
+
+        @staticmethod
+        def is_ascii_printable(s):
+            return all(32 <= ord(char) < 127 for char in s)
+
+        preprocess_map = {
+            "base64": preprocess_base64,
+            "hex": preprocess_hex,
+            "url-encoded": preprocess_urlencoded,
+        }
+        postprocess_map = {
+            "base64": postprocess_base64,
+            "hex": postprocess_hex,
+            "url-encoded": postprocess_urlencoded,
+        }
+
+        # Format-specific functions for isolating and updating parameters
+        format_isolate_map = {
+            "json": lambda self: self.isolate_parameter(),
+            "xml": lambda self: self.isolate_parameter(),
+        }
+        format_update_map = {
+            "json": lambda self, value: self.update_json_parameter(value),
+            "xml": lambda self, value: self.update_xml_parameter(value),  # Placeholder
+        }
+
+        def initialize_value(self, value=None):
+            self.envelopes, end_format_dict = self.recurse_envelopes(value)
+            if self.envelopes:
+                log.debug(f"Discovered the following envelopes: [{','.join(self.envelopes)}]")
+
+            if end_format_dict is not None:
+                self.end_format_type = list(end_format_dict.keys())[0]
+                log.debug(f"Identified the following end format: [{self.end_format_type}]")
+                self.end_format_data = list(end_format_dict.values())[0]
+            else:
+                self.end_format_type = None
+                self.end_format_data = None
+            self.end_format_subparameter = None
+
+        def remove_envelopes(self, value):
+            """
+            Remove envelopes from the value, processing each envelope in the order it was applied.
+            If the final format is present, trigger the appropriate handler (e.g., for JSON).
+            """
+            # Apply the preprocess functions in the order the envelopes were applied
+            for env in self.envelopes:
+                func = self.preprocess_map.get(env)
+                if func:
+                    value = func(value)
+
+            # Dynamically select the appropriate isolate function based on the final format
+            isolate_func = self.format_isolate_map.get(self.end_format_type)
+            if isolate_func:
+                return isolate_func(self)
+
+            return value
+
+        def add_envelopes(self, value):
+            """
+            Add envelopes back to the value, processing in reverse order.
+            If the final format is present, trigger the appropriate handler (e.g., for JSON).
+            """
+            # Dynamically select the appropriate update function based on the final format
+            update_func = self.format_update_map.get(self.end_format_type)
+            if update_func:
+                value = update_func(self, value)
+
+            # Apply the envelopes in reverse order
+            for env in self.envelopes[::-1]:
+                func = self.postprocess_map.get(env)
+                if func:
+                    value = func(value)
+            return value
+
+        def recurse_envelopes(self, value, envelopes=None, end_format=None):
+            if envelopes is None:
+                envelopes = []
+            log.debug(
+                f"Starting recurse with value: {value}, current envelopes: {', '.join(envelopes)}, current end format: {end_format}"
+            )
+
+            if value is None or value == "" or isinstance(value, int):
+                return envelopes, end_format
+
+            # Try URL decoding
+            try:
+                decoded_url = unquote(value)
+                if decoded_url != value and self.is_ascii_printable(decoded_url):
+                    envelopes.append("url-encoded")
+                    envelopes, end_format_dict = self.recurse_envelopes(decoded_url, envelopes)
+                    return envelopes, end_format_dict
+            except Exception:
+                pass  # Not valid URL encoding
+
+            # Try base64 decoding
+            try:
+                decoded_base64 = base64.b64decode(value).decode()
+                if self.is_ascii_printable(decoded_base64):
+                    envelopes.append("base64")
+                    envelopes, end_format_dict = self.recurse_envelopes(decoded_base64, envelopes)
+                    return envelopes, end_format_dict
+            except (binascii.Error, UnicodeDecodeError):
+                pass  # Not valid base64
+
+            # Try hex decoding
+            try:
+                decoded_hex = bytes.fromhex(value).decode("utf-8")
+                if self.is_ascii_printable(decoded_hex):
+                    envelopes.append("hex")
+                    envelopes, end_format_dict = self.recurse_envelopes(decoded_hex, envelopes)
+                    return envelopes, end_format_dict
+            except (ValueError, UnicodeDecodeError):
+                pass  # Not valid hex
+
+            # Try JSON parsing
+            try:
+                decoded_json = json.loads(value)
+                if isinstance(decoded_json, (str, dict, list)):
+                    return envelopes, {"json": decoded_json}
+            except json.JSONDecodeError:
+                pass  # Not valid JSON
+
+            # Try XML parsing
+            try:
+                decoded_xml = ET.fromstring(value)
+                # Pass 'decoded_xml' to 'xml_to_dict'
+                xml_dict = self.xml_to_dict(decoded_xml)  # Pass decoded XML as the 'elem' argument
+                return envelopes, {"xml": xml_dict}  # Store as JSON-like dict, not XML
+            except ET.ParseError:
+                pass  # Not valid XML
+
+            return envelopes, end_format
+
+        def isolate_parameter(self):
+            """
+            Isolate the specified subparameter from the data structure (JSON/XML).
+            The subparameter is accessed using dot notation for nested keys.
+            """
+            if self.end_format_data and self.end_format_subparameter:
+                # Split the dot notation string into keys
+                keys = self.end_format_subparameter.split(".")
+
+                # Traverse the nested structure using the keys
+                subparameter_value = self.end_format_data
+                for key in keys:
+                    if isinstance(subparameter_value, dict):
+                        subparameter_value = subparameter_value.get(key)
+                    else:
+                        # If the structure is broken (not a dict), return None
+                        return None
+
+                return subparameter_value
+
+            return None
+
+        def update_json_parameter(self, new_value):
+            """
+            Update the specified subparameter in the JSON structure and rebuild it.
+            """
+            # Work with a copy to avoid modifying the original `end_format_data`
+            end_format_data_copy = deepcopy(self.end_format_data)
+
+            if end_format_data_copy:
+                end_format_data_copy[self.end_format_subparameter] = new_value
+                return json.dumps(end_format_data_copy)
+            return new_value
+
+        def update_xml_parameter(self, new_value):
+            """
+            Convert the JSON-like structure back into an XML string after updating the specific parameter.
+            """
+            if self.end_format_data and self.end_format_subparameter:
+                # Split the dot notation into keys
+                keys = self.end_format_subparameter.split(".")
+
+                # Traverse the nested dictionary using the keys to find the target subparameter
+                current_data = self.end_format_data
+                for key in keys[:-1]:  # Traverse up to the second-to-last key
+                    current_data = current_data.get(key, {})
+
+                # Update the target subparameter with the new value
+                if isinstance(current_data, dict):
+                    current_data[keys[-1]] = new_value
+
+                # Convert the JSON-like dict back to an XML string
+                return self.dict_to_xml(self.end_format_data)
+
+            return new_value
+
+        def to_dict(self):
+            return {
+                "envelopes": self.envelopes,
+                "end_format_type": self.end_format_type,
+                "end_format_data": self.end_format_data,
+                "end_format_subparameter": self.end_format_subparameter,
+            }
+
+        def __getstate__(self):
+            return self.to_dict()
+
+        def __str__(self):
+            return f"ParameterEnvelopes(envelopes={self.envelopes}, end_format_type={self.end_format_type}, end_format_data={self.end_format_data},  end_format_subparameter={self.end_format_subparameter})"
+
+        __repr__ = __str__
+
+    @classmethod
+    def from_dict(cls, data):
+        instance = cls()
+        instance.envelopes = data.get("envelopes", [])
+        instance.end_format_type = data.get("end_format_type")
+        instance.end_format_data = data.get("end_format_data")
+        instance.end_format_subparameter = data.get("end_format_subparameter")
+        return instance
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if "original_value" in self.data.keys():
+            parameterEnvelope_instance = self.ParameterEnvelopes()
+            parameterEnvelope_instance.initialize_value(self.data["original_value"])
+            setattr(self, "envelopes", parameterEnvelope_instance)
+
+            envelopes = getattr(self, "envelopes", None)
+            if (
+                envelopes is not None
+                and getattr(envelopes, "end_format_type", None) is not None
+                and getattr(envelopes, "end_format_data", None)
+            ):
+                end_format_data = envelopes.end_format_data
+
+                def extract_keys_with_values(data, parent_key=""):
+                    """
+                    Recursively extract all keys from nested dictionaries that have values (non-empty).
+                    Construct a path-like structure with dot notation (e.g., 'find.search').
+                    """
+                    keys = []
+                    if isinstance(data, dict):
+                        for key, value in data.items():
+                            # Construct the full key path using dot notation
+                            full_key = f"{parent_key}.{key}" if parent_key else key
+
+                            # Only add keys that have non-empty values
+                            if value:
+                                if isinstance(value, dict):
+                                    # Recursively check nested dictionaries
+                                    keys.extend(extract_keys_with_values(value, full_key))
+                                else:
+                                    # Add the key if it has a non-empty value
+                                    keys.append(full_key)
+                    return keys
+
+                # Extract all keys that have non-empty values
+                end_format_data_keys = extract_keys_with_values(end_format_data)
+                # If there are keys, assign the first key to end_format_subparameter
+                if end_format_data_keys:
+
+                    # Assign the first key to end_format_subparameter
+                    setattr(envelopes, "end_format_subparameter", end_format_data_keys[0])
+                    setattr(envelopes, "end_format_subparameter", end_format_data_keys[0])
+
+                    # Iterate through the remaining keys, starting from the second one
+                    for p in end_format_data_keys[1:]:
+                        log.debug(f"generating copy of event for subparameter {p} of type {envelopes.end_format_type}")
+
+                        # Make a copy of the current event data
+                        cloned_event = self.clone()
+                        cloned_envelopes = getattr(cloned_event, "envelopes")
+                        cloned_envelopes.end_format_subparameter = p
+                        asyncio.run_coroutine_threadsafe(
+                            self.module.emit_event(cloned_event), asyncio.get_event_loop()
+                        )
+
     def _data_id(self):
         # dedupe by url:name:param_type
         url = self.data.get("url", "")
         name = self.data.get("name", "")
         param_type = self.data.get("type", "")
-        return f"{url}:{name}:{param_type}"
+        envelopes = getattr(self, "envelopes", None)
+        subparameter = getattr(envelopes, "end_format_subparameter", "") if envelopes else ""
+
+        return f"{url}:{name}:{param_type}:{subparameter}"
+
+    def _outgoing_dedup_hash(self, event):
+
+        envelopes = self.data.get("envelopes")
+        return hash(
+            (
+                str(event.host),
+                event.data["url"],
+                event.data.get("name", ""),
+                event.data.get("type", ""),
+                event.data.get("envelopes", ""),
+            )
+        )
 
     def _url(self):
         return self.data["url"]
@@ -1651,7 +2050,6 @@ def make_event(
             tags.add("affiliate")
 
         event_class = globals().get(event_type, DefaultEvent)
-
         return event_class(
             data,
             event_type=event_type,
@@ -1697,7 +2095,6 @@ def event_from_json(j, siem_friendly=False):
             "tags": j.get("tags", []),
             "confidence": j.get("confidence", 100),
             "context": j.get("discovery_context", None),
-            "dummy": True,
         }
         if siem_friendly:
             data = j["data"][event_type]
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 4e45ce21e6..6cb3bf9138 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -3,6 +3,7 @@
 import copy
 import json
 import math
+import base64
 import random
 import string
 import asyncio
@@ -11,6 +12,7 @@
 import ahocorasick
 import regex as re
 import subprocess as sp
+
 from pathlib import Path
 from contextlib import suppress
 from unidecode import unidecode  # noqa F401
diff --git a/bbot/core/helpers/regexes.py b/bbot/core/helpers/regexes.py
index 24d53286f5..1bbb2926a8 100644
--- a/bbot/core/helpers/regexes.py
+++ b/bbot/core/helpers/regexes.py
@@ -116,8 +116,12 @@
 
 # For use with excavate paramaters extractor
 input_tag_regex = re.compile(
-    r"<input[^>]+?name=[\"\']?([\.$\w]+)[\"\']?(?:[^>]*?value=[\"\']([=+\/\w]*)[\"\'])?[^>]*>"
+    r"<input[^>]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?value=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>"
 )
+input_tag_regex2 = re.compile(
+    r"<input[^>]*?value=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?name=[\"\']?([\._=+\/\w]+)[\"\']?[^>]*?>"
+)
+input_tag_novalue_regex = re.compile(r"<input(?![^>]*\bvalue=)[^>]*?name=[\"\']?([\._=+\/\w]*)[\"\']?[^>]*?>")
 # jquery_get_regex = re.compile(r"url:\s?[\"\'].+?\?(\w+)=")
 # jquery_get_regex = re.compile(r"\$.get\([\'\"].+[\'\"].+\{(.+)\}")
 # jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}")
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 481b63f589..0314986306 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -432,11 +432,13 @@ def extract(self):
         class GetForm(ParameterExtractorRule):
             name = "GET Form"
             discovery_regex = r'/<form[^>]*\bmethod=["\']?get["\']?[^>]*>.*<\/form>/s nocase'
-            form_content_regexes = [
-                bbot_regexes.input_tag_regex,
-                bbot_regexes.select_tag_regex,
-                bbot_regexes.textarea_tag_regex,
-            ]
+            form_content_regexes = {
+                "input_tag_regex": bbot_regexes.input_tag_regex,
+                "input_tag_regex2": bbot_regexes.input_tag_regex2,
+                "input_tag_novalue_regex": bbot_regexes.input_tag_novalue_regex,
+                "select_tag_regex": bbot_regexes.select_tag_regex,
+                "textarea_tag_regex": bbot_regexes.textarea_tag_regex,
+            }
             extraction_regex = bbot_regexes.get_form_regex
             output_type = "GETPARAM"
 
@@ -448,16 +450,26 @@ def extract(self):
                         form_action = form_action.lstrip(".")
 
                     form_parameters = {}
-                    for form_content_regex in self.form_content_regexes:
+                    match_found = False
+                    for form_content_regex_name, form_content_regex in self.form_content_regexes.items():
                         input_tags = form_content_regex.findall(form_content)
+                        if input_tags:
+                            match_found = True
 
-                        for parameter_name, original_value in input_tags:
-                            form_parameters[parameter_name] = original_value.strip()
+                            if form_content_regex_name == "input_tag_novalue_regex":
+                                form_parameters[input_tags[0]] = None
 
-                        for parameter_name, original_value in form_parameters.items():
-                            yield self.output_type, parameter_name, original_value, form_action, _exclude_key(
-                                form_parameters, parameter_name
-                            )
+                            else:
+                                if form_content_regex_name == "input_tag_regex2":
+                                    input_tags = input_tags = [(b, a) for a, b in input_tags]
+
+                                for parameter_name, original_value in input_tags:
+                                    form_parameters[parameter_name] = original_value.strip()
+
+                            for parameter_name, original_value in form_parameters.items():
+                                yield self.output_type, parameter_name, original_value, form_action, _exclude_key(
+                                    form_parameters, parameter_name
+                                )
 
         class GetForm2(GetForm):
             extraction_regex = bbot_regexes.get_form_regex2
@@ -515,12 +527,12 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte
                                 f"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule"
                             )
                             endpoint = event.data["url"] if not endpoint else endpoint
+                            path = f"{event.parsed_url.path.lstrip('/')}"
                             url = (
                                 endpoint
                                 if endpoint.startswith(("http://", "https://"))
-                                else f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{endpoint}"
+                                else f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{path}{endpoint}"
                             )
-
                             if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):
 
                                 if self.excavate.in_bl(parameter_name) == False:
@@ -1003,8 +1015,8 @@ async def handle_event(self, event):
                             self.debug(f"Cookie found without '=': {header_value}")
                             continue
                         else:
-                            cookie_name = header_value.split("=")[0]
-                            cookie_value = header_value.split("=")[1].split(";")[0]
+                            cookie_name, _, remainder = header_value.partition("=")
+                            cookie_value = remainder.split(";")[0]
 
                             if self.in_bl(cookie_name) == False:
                                 self.assigned_cookies[cookie_name] = cookie_value
diff --git a/bbot/modules/lightfuzz.py b/bbot/modules/lightfuzz.py
index 1f724eb9a1..b1fcd19b27 100644
--- a/bbot/modules/lightfuzz.py
+++ b/bbot/modules/lightfuzz.py
@@ -128,6 +128,16 @@ async def run_submodule(self, submodule, event):
         if len(submodule_instance.results) > 0:
             for r in submodule_instance.results:
                 event_data = {"host": str(event.host), "url": event.data["url"], "description": r["description"]}
+
+                envelopes = getattr(event, "envelopes", None)
+                if envelopes and envelopes.envelopes:
+                    envelope_summary = f'[{"->".join(envelopes.envelopes)}]'
+                    if envelopes.end_format_type:
+                        envelope_summary += f" Format: [{envelopes.end_format_type}] with subparameter [{envelopes.end_format_subparameter}])"
+
+                    # Append the envelope summary to the description
+                    event_data["description"] += f" Envelopes: {envelope_summary}"
+
                 if r["type"] == "VULNERABILITY":
                     event_data["severity"] = r["severity"]
                 await self.emit_event(
diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py
index 5d15f1a190..31438592be 100644
--- a/bbot/modules/lightfuzz_submodules/base.py
+++ b/bbot/modules/lightfuzz_submodules/base.py
@@ -1,4 +1,5 @@
 import copy
+import base64
 
 
 class BaseLightfuzz:
@@ -19,9 +20,10 @@ def additional_params_process(self, additional_params, additional_params_populat
         return new_additional_params
 
     async def send_probe(self, probe):
+        probe = self.probe_value_outgoing(probe)
         getparams = {self.event.data["name"]: probe}
         url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl()
-        self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}")
+        self.lightfuzz.critical(f"lightfuzz sending probe with URL: {url}")
         r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10)
         if r:
             return r.text
@@ -29,7 +31,7 @@ async def send_probe(self, probe):
     def compare_baseline(
         self, event_type, probe, cookies, additional_params_populate_empty=False, speculative_mode="GETPARAM"
     ):
-
+        probe = self.probe_value_outgoing(probe)
         http_compare = None
 
         if event_type == "SPECULATIVE":
@@ -55,7 +57,7 @@ def compare_baseline(
                 self.event.data["url"], include_cache_buster=False, headers=headers, cookies=cookies
             )
         elif event_type == "POSTPARAM":
-            data = {self.event.data["name"]: f"{probe}"}
+            data = {self.probe_value_outgoing(self.event.data["name"]): f"{probe}"}
             if self.event.data["additional_params"] is not None:
                 data.update(
                     self.additional_params_process(
@@ -67,6 +69,21 @@ def compare_baseline(
             )
         return http_compare
 
+    async def baseline_probe(self, cookies):
+        if self.event.data.get("eventtype") == "POSTPARAM":
+            method = "POST"
+        else:
+            method = "GET"
+
+        return await self.lightfuzz.helpers.request(
+            method=method,
+            cookies=cookies,
+            url=self.event.data.get("url"),
+            allow_redirects=False,
+            retries=1,
+            timeout=10,
+        )
+
     async def compare_probe(
         self,
         http_compare,
@@ -77,6 +94,8 @@ async def compare_probe(
         additional_params_override={},
         speculative_mode="GETPARAM",
     ):
+
+        probe = self.probe_value_outgoing(probe)
         additional_params = copy.deepcopy(self.event.data.get("additional_params", {}))
         if additional_params_override:
             for k, v in additional_params_override.items():
@@ -109,18 +128,20 @@ async def standard_probe(
         self,
         event_type,
         cookies,
-        probe_value,
+        probe,
         timeout=10,
         additional_params_populate_empty=False,
         speculative_mode="GETPARAM",
     ):
 
+        probe = self.probe_value_outgoing(probe)
+
         if event_type == "SPECULATIVE":
             event_type = speculative_mode
 
         method = "GET"
         if event_type == "GETPARAM":
-            url = f"{self.event.data['url']}?{self.event.data['name']}={probe_value}"
+            url = f"{self.event.data['url']}?{self.event.data['name']}={probe}"
             if "additional_params" in self.event.data.keys() and self.event.data["additional_params"] is not None:
                 url = self.lightfuzz.helpers.add_get_params(
                     url, self.event.data["additional_params"], encode=False
@@ -128,15 +149,15 @@ async def standard_probe(
         else:
             url = self.event.data["url"]
         if event_type == "COOKIE":
-            cookies_probe = {self.event.data["name"]: probe_value}
+            cookies_probe = {self.event.data["name"]: probe}
             cookies = {**cookies, **cookies_probe}
         if event_type == "HEADER":
-            headers = {self.event.data["name"]: probe_value}
+            headers = {self.event.data["name"]: probe}
         else:
             headers = {}
         if event_type == "POSTPARAM":
             method = "POST"
-            data = {self.event.data["name"]: probe_value}
+            data = {self.event.data["name"]: probe}
             if self.event.data["additional_params"] is not None:
                 data.update(
                     self.additional_params_process(
@@ -166,10 +187,22 @@ def metadata(self):
             )
         return metadata_string
 
-    def probe_value(self, populate_empty=True):
+    def probe_value_incoming(self, populate_empty=True):
 
         probe_value = str(self.event.data.get("original_value", ""))
         if (probe_value is None or len(probe_value) == 0) and populate_empty == True:
             probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True)
-
+        self.lightfuzz.debug(f"probe_value_incoming (before modification): {probe_value}")
+        envelopes_instance = getattr(self.event, "envelopes", None)
+        self.lightfuzz.hugesuccess(envelopes_instance)
+        probe_value = envelopes_instance.remove_envelopes(probe_value)
+        self.lightfuzz.debug(f"probe_value_incoming (after modification): {probe_value}")
         return probe_value
+
+    def probe_value_outgoing(self, outgoing_probe_value):
+        self.lightfuzz.debug(f"probe_value_outgoing (before modification): {outgoing_probe_value}")
+        envelopes_instance = getattr(self.event, "envelopes", None)
+        self.lightfuzz.hugesuccess(envelopes_instance)
+        outgoing_probe_value = envelopes_instance.add_envelopes(outgoing_probe_value)
+        self.lightfuzz.debug(f"probe_value_outgoing (after modification): {outgoing_probe_value}")
+        return outgoing_probe_value
diff --git a/bbot/modules/lightfuzz_submodules/cmdi.py b/bbot/modules/lightfuzz_submodules/cmdi.py
index 92385c6434..86504ac74b 100644
--- a/bbot/modules/lightfuzz_submodules/cmdi.py
+++ b/bbot/modules/lightfuzz_submodules/cmdi.py
@@ -7,9 +7,8 @@
 class CmdILightfuzz(BaseLightfuzz):
 
     async def fuzz(self):
-
         cookies = self.event.data.get("assigned_cookies", {})
-        probe_value = self.probe_value()
+        probe_value = self.probe_value_incoming()
 
         canary = self.lightfuzz.helpers.rand_string(8, numeric_only=True)
         http_compare = self.compare_baseline(self.event.data["type"], probe_value, cookies)
diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py
index 9141c240d8..da7e52a63c 100644
--- a/bbot/modules/lightfuzz_submodules/crypto.py
+++ b/bbot/modules/lightfuzz_submodules/crypto.py
@@ -194,7 +194,7 @@ async def padding_oracle(self, probe_value, cookies):
                 }
             )
 
-    async def error_string_search(self, text_dict):
+    async def error_string_search(self, text_dict, baseline_text):
 
         matching_techniques = set()
         matching_strings = set()
@@ -206,12 +206,18 @@ async def error_string_search(self, text_dict):
             matching_techniques.add(label)
         context = f"Lightfuzz Cryptographic Probe Submodule detected a cryptographic error after manipulating parameter: [{self.event.data['name']}]"
         if len(matching_strings) > 0:
-            self.results.append(
-                {
-                    "type": "FINDING",
-                    "description": f"Possible Cryptographic Error. {self.metadata()} Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]",
-                    "context": context,
-                }
+            false_positive_check = self.lightfuzz.helpers.string_scan(self.crypto_error_strings, baseline_text)
+            false_positive_matches = set(matched_strings) & set(false_positive_check)
+            if not false_positive_matches:
+                self.results.append(
+                    {
+                        "type": "FINDING",
+                        "description": f"Possible Cryptographic Error. {self.metadata()} Strings: [{','.join(matching_strings)}] Detection Technique(s): [{','.join(matching_techniques)}]",
+                        "context": context,
+                    }
+                )
+            self.lightfuzz.debug(
+                f"Aborting cryptographic error reporting - baseline_text already contained detected string(s) ({','.join(false_positive_check)})"
             )
 
     @staticmethod
@@ -229,13 +235,20 @@ def identify_hash_function(hash_bytes):
             return hash_functions[hash_length]
 
     async def fuzz(self):
+
         cookies = self.event.data.get("assigned_cookies", {})
-        probe_value = self.probe_value(populate_empty=False)
+        probe_value = self.probe_value_incoming(populate_empty=False)
         if not probe_value:
             self.lightfuzz.debug(
                 f"The Cryptography Probe Submodule requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]"
             )
             return
+
+        baseline_probe = await self.baseline_probe(cookies)
+        if not baseline_probe:
+            self.lightfuzz.critical(f"Couldn't get baseline_probe for url {self.event.data['url']}, aborting")
+            return
+
         try:
             truncate_probe_value = self.modify_string(probe_value, action="truncate")
             mutate_probe_value = self.modify_string(probe_value, action="mutate")
@@ -290,7 +303,7 @@ async def fuzz(self):
 
         # Cryptographic Error String Test
         await self.error_string_search(
-            {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text}
+            {"truncate value": truncate_probe[3].text, "mutate value": mutate_probe[3].text}, baseline_probe.text
         )
 
         if confirmed_techniques or (
diff --git a/bbot/modules/lightfuzz_submodules/path.py b/bbot/modules/lightfuzz_submodules/path.py
index d4834221d0..ab63d278ba 100644
--- a/bbot/modules/lightfuzz_submodules/path.py
+++ b/bbot/modules/lightfuzz_submodules/path.py
@@ -9,7 +9,7 @@ class PathTraversalLightfuzz(BaseLightfuzz):
 
     async def fuzz(self):
         cookies = self.event.data.get("assigned_cookies", {})
-        probe_value = self.probe_value(populate_empty=False)
+        probe_value = self.probe_value_incoming(populate_empty=False)
         if not probe_value:
             self.lightfuzz.debug(
                 f"Path Traversal detection requires original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]"
diff --git a/bbot/modules/lightfuzz_submodules/serial.py b/bbot/modules/lightfuzz_submodules/serial.py
index 6081a550d0..c9939f6f7b 100644
--- a/bbot/modules/lightfuzz_submodules/serial.py
+++ b/bbot/modules/lightfuzz_submodules/serial.py
@@ -4,7 +4,6 @@
 
 class SerialLightfuzz(BaseLightfuzz):
     async def fuzz(self):
-
         cookies = self.event.data.get("assigned_cookies", {})
         control_payload = "DEADBEEFCAFEBABE1234567890ABCDEF"
         serialization_payloads = {
@@ -27,7 +26,7 @@ async def fuzz(self):
             "java.io.optionaldataexception",
         ]
 
-        probe_value = self.probe_value(populate_empty=False)
+        probe_value = self.probe_value_incoming(populate_empty=False)
         if probe_value:
             self.lightfuzz.debug(
                 f"The Serialization Submodule only operates when there if no original value, aborting [{self.event.data['type']}] [{self.event.data['name']}]"
diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py
index 9d5db9518e..ac660fc4cb 100644
--- a/bbot/modules/lightfuzz_submodules/sqli.py
+++ b/bbot/modules/lightfuzz_submodules/sqli.py
@@ -29,14 +29,7 @@ def evaluate_delay(self, mean_baseline, measured_delay):
     async def fuzz(self):
 
         cookies = self.event.data.get("assigned_cookies", {})
-
-        # custom probe_value generation
-        if "original_value" in self.event.data and self.event.data["original_value"] is not None:
-            probe_value = urllib.parse.quote(str(self.event.data["original_value"]), safe="")
-
-        else:
-            probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True)
-
+        probe_value = self.probe_value_incoming(populate_empty=True)
         http_compare = self.compare_baseline(
             self.event.data["type"], probe_value, cookies, additional_params_populate_empty=True
         )
diff --git a/bbot/presets/web/lightfuzz-intense.yml b/bbot/presets/web/lightfuzz-intense.yml
index 16caecbcab..a16c43872f 100644
--- a/bbot/presets/web/lightfuzz-intense.yml
+++ b/bbot/presets/web/lightfuzz-intense.yml
@@ -19,6 +19,6 @@ config:
   modules:
     lightfuzz:
       force_common_headers: False
-      submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss]
+      enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss]
     excavate:
       retain_querystring: True
\ No newline at end of file
diff --git a/bbot/presets/web/lightfuzz-max.yml b/bbot/presets/web/lightfuzz-max.yml
index e0c7a738ba..0b8fbb23c6 100644
--- a/bbot/presets/web/lightfuzz-max.yml
+++ b/bbot/presets/web/lightfuzz-max.yml
@@ -19,6 +19,6 @@ config:
   modules:
     lightfuzz:
       force_common_headers: True
-      submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss]
+      enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss]
     excavate:
       retain_querystring: True
\ No newline at end of file
diff --git a/bbot/presets/web/lightfuzz-xss.yml b/bbot/presets/web/lightfuzz-xss.yml
index fd13b2c67e..a0162bce29 100644
--- a/bbot/presets/web/lightfuzz-xss.yml
+++ b/bbot/presets/web/lightfuzz-xss.yml
@@ -12,4 +12,4 @@ config:
     spider_depth: 5
   modules:
     lightfuzz:
-      submodules_enabled: [xss]
\ No newline at end of file
+      enabled_submodules: [xss]
\ No newline at end of file
diff --git a/bbot/presets/web/lightfuzz.yml b/bbot/presets/web/lightfuzz.yml
index 85cce074b7..bde8e7a565 100644
--- a/bbot/presets/web/lightfuzz.yml
+++ b/bbot/presets/web/lightfuzz.yml
@@ -13,7 +13,7 @@ config:
     spider_depth: 4
   modules:
     lightfuzz:
-      submodules_enabled: [cmdi,crypto,path,serial,sqli,ssti,xss]
+      enabled_submodules: [cmdi,crypto,path,serial,sqli,ssti,xss]
       disable_post: True
     excavate:
       retain_querystring: True
diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py
index 71ab9fd64e..551dd3b031 100644
--- a/bbot/test/test_step_2/module_tests/test_module_excavate.py
+++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py
@@ -636,6 +636,29 @@ def check(self, module_test, events):
         assert excavate_xml_extraction, "Excavate failed to extract xml parameter"
 
 
+class TestExcavateParameterExtraction_inputtagnovalue(ModuleTestBase):
+
+    targets = ["http://127.0.0.1:8888/"]
+
+    # hunt is added as parameter extraction is only activated by one or more modules that consume WEB_PARAMETER
+    modules_overrides = ["httpx", "excavate", "hunt"]
+    getparam_extract_html = """
+<form action=/ method=GET><input type=text name="novalue"><button type=submit class=button>Submit</button></form>
+    """
+
+    async def setup_after_prep(self, module_test):
+        respond_args = {"response_data": self.getparam_extract_html, "headers": {"Content-Type": "text/html"}}
+        module_test.set_expect_requests(respond_args=respond_args)
+
+    def check(self, module_test, events):
+        excavate_getparam_extraction = False
+        for e in events:
+            if e.type == "WEB_PARAMETER":
+                if "HTTP Extracted Parameter [novalue] (GET Form Submodule)":
+                    excavate_getparam_extraction = True
+        assert excavate_getparam_extraction, "Excavate failed to extract web parameter"
+
+
 class excavateTestRule(ExcavateRule):
     yara_rules = {
         "SearchForText": 'rule SearchForText { meta: description = "Contains the text AAAABBBBCCCC" strings: $text = "AAAABBBBCCCC" condition: $text }',
diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
index 4c54721306..eb5607085e 100644
--- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
+++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
@@ -1,9 +1,13 @@
+import json
 import re
+import base64
 
 from .base import ModuleTestBase, tempwordlist
 from werkzeug.wrappers import Response
 from urllib.parse import unquote
 
+import xml.etree.ElementTree as ET
+
 
 # Path Traversal single dot tolerance
 class Test_Lightfuzz_path_singledot(ModuleTestBase):
@@ -231,6 +235,212 @@ def check(self, module_test, events):
         assert xss_finding_emitted, "Between Tags XSS FINDING not emitted"
 
 
+# Base64 Envelope XSS Detection
+class Test_Lightfuzz_envelope_base64(Test_Lightfuzz_xss):
+    def request_handler(self, request):
+
+        qs = str(request.query_string.decode())
+
+        print("****")
+        print(qs)
+
+        parameter_block = """
+        <section class=search>
+            <form action=/ method=GET>
+                <input type=text value='dGV4dA==' name=search>
+                <button type=submit class=button>Search</button>
+            </form>
+        </section>
+        """
+        if "search=" in qs:
+            value = qs.split("search=")[1]
+            if "&" in value:
+                value = value.split("&")[0]
+            xss_block = f"""
+        <section class=blog-header>
+            <h1>0 search results for '{unquote(base64.b64decode(value))}'</h1>
+            <hr>
+        </section>
+        """
+            print("XSS BLOCK:")
+            print(xss_block)
+            return Response(xss_block, status=200)
+        return Response(parameter_block, status=200)
+
+    def check(self, module_test, events):
+
+        web_parameter_emitted = False
+        xss_finding_emitted = False
+        for e in events:
+            if e.type == "WEB_PARAMETER":
+                if "HTTP Extracted Parameter [search]" in e.data["description"]:
+                    web_parameter_emitted = True
+
+            if e.type == "FINDING":
+                if (
+                    "Possible Reflected XSS. Parameter: [search] Context: [Between Tags (z tag)"
+                    in e.data["description"]
+                ):
+                    xss_finding_emitted = True
+
+        assert web_parameter_emitted, "WEB_PARAMETER was not emitted"
+        assert xss_finding_emitted, "Between Tags XSS FINDING not emitted"
+
+
+# Hex Envelope XSS Detection
+class Test_Lightfuzz_envelope_hex(Test_Lightfuzz_envelope_base64):
+    def request_handler(self, request):
+        qs = str(request.query_string.decode())
+
+        parameter_block = """
+        <section class=search>
+            <form action=/ method=GET>
+                <input type=text value='7b22736561726368223a202264656d6f6b6579776f7264227d' name=search>
+                <button type=submit class=button>Search</button>
+            </form>
+        </section>
+        """
+
+        if "search=" in qs:
+            value = qs.split("search=")[1]
+            if "&" in value:
+                value = value.split("&")[0]
+
+            try:
+                # Decode the hex value
+                decoded_value = bytes.fromhex(unquote(value)).decode()
+                print(f"Decoded hex value: {decoded_value}")
+
+                # Parse the decoded value as JSON
+                json_data = json.loads(decoded_value)
+
+                # Extract the desired parameter from the JSON (e.g., 'search')
+                if "search" in json_data:
+                    extracted_value = json_data["search"]
+                else:
+                    extracted_value = "[Parameter not found in JSON]"
+
+            except (json.JSONDecodeError, ValueError) as e:
+                extracted_value = "[Invalid hex or JSON format]"
+
+            xss_block = f"""
+        <section class=blog-header>
+            <h1>0 search results for '{extracted_value}'</h1>
+            <hr>
+        </section>
+        """
+            print("XSS BLOCK:")
+            print(xss_block)
+            return Response(xss_block, status=200)
+
+        return Response(parameter_block, status=200)
+
+
+# Base64 (JSON) Envelope XSS Detection
+class Test_Lightfuzz_envelope_jsonb64(Test_Lightfuzz_envelope_base64):
+    def request_handler(self, request):
+        qs = str(request.query_string.decode())
+
+        print("****")
+        print(qs)
+
+        parameter_block = """
+        <section class=search>
+            <form action=/ method=GET>
+                <input type=text value='eyJzZWFyY2giOiAiZGVtb2tleXdvcmQifQ==' name=search>
+                <button type=submit class=button>Search</button>
+            </form>
+        </section>
+        """
+
+        if "search=" in qs:
+            value = qs.split("search=")[1]
+            if "&" in value:
+                value = value.split("&")[0]
+
+            try:
+                # Base64 decode the value
+                decoded_value = base64.b64decode(unquote(value)).decode()
+                print(f"Decoded base64 value: {decoded_value}")
+
+                # Parse the decoded value as JSON
+                json_data = json.loads(decoded_value)
+
+                # Extract the desired parameter from the JSON (e.g., 'search')
+                if "search" in json_data:
+                    extracted_value = json_data["search"]
+                else:
+                    extracted_value = "[Parameter not found in JSON]"
+
+            except (json.JSONDecodeError, base64.binascii.Error) as e:
+                extracted_value = "[Invalid base64 or JSON format]"
+
+            xss_block = f"""
+        <section class=blog-header>
+            <h1>0 search results for '{extracted_value}'</h1>
+            <hr>
+        </section>
+        """
+            print("XSS BLOCK:")
+            print(xss_block)
+            return Response(xss_block, status=200)
+
+        return Response(parameter_block, status=200)
+
+
+# Base64 (XML) Envelope XSS Detection
+class Test_Lightfuzz_envelope_xmlb64(Test_Lightfuzz_envelope_base64):
+    def request_handler(self, request):
+        qs = str(request.query_string.decode())
+
+        print("****")
+        print(qs)
+
+        parameter_block = """
+        <section class=search>
+            <form action=/ method=GET>
+                <input type=text value='PGZpbmQ+PHNlYXJjaD5kZW1va2V5d29yZDwvc2VhcmNoPjwvZmluZD4=' name=search>
+                <button type=submit class=button>Search</button>
+            </form>
+        </section>
+        """
+
+        if "search=" in qs:
+            value = qs.split("search=")[1]
+            if "&" in value:
+                value = value.split("&")[0]
+
+            try:
+                # Base64 decode the value
+                decoded_value = base64.b64decode(unquote(value)).decode()
+                print(f"Decoded base64 value: {decoded_value}")
+
+                # Parse the decoded value as XML
+                root = ET.fromstring(decoded_value)
+
+                # Extract the desired parameter from the XML (e.g., 'search')
+                search_element = root.find(".//search")
+                if search_element is not None:
+                    extracted_value = search_element.text
+                else:
+                    extracted_value = "[Parameter not found in XML]"
+
+            except (ET.ParseError, base64.binascii.Error) as e:
+                extracted_value = "[Invalid base64 or XML format]"
+
+            xss_block = f"""
+        <section class=blog-header>
+            <h1>0 search results for '{extracted_value}'</h1>
+            <hr>
+        </section>
+        """
+            print("XSS BLOCK:")
+            print(xss_block)
+            return Response(xss_block, status=200)
+
+        return Response(parameter_block, status=200)
+
+
 # In Tag Attribute XSS Detection
 class Test_Lightfuzz_xss_intag(Test_Lightfuzz_xss):
     def request_handler(self, request):
@@ -1101,3 +1311,110 @@ def check(self, module_test, events):
 
         assert excavate_json_extraction, "Excavate failed to extract json parameter"
         assert xss_finding_emitted, "Between Tags XSS FINDING not emitted"
+
+
+class Test_Lightfuzz_crypto_error(ModuleTestBase):
+
+    targets = ["http://127.0.0.1:8888/"]
+    modules_overrides = ["httpx", "excavate", "lightfuzz"]
+    config_overrides = {
+        "interactsh_disable": True,
+        "modules": {
+            "lightfuzz": {"enabled_submodules": ["crypto"]},
+        },
+    }
+
+    def request_handler(self, request):
+
+        qs = str(request.query_string.decode())
+
+        parameter_block = """
+        <section class=secret>
+            <form action=/ method=GET>
+                <input type=text value='08a5a2cea9c5a5576e6e5314edcba581d21c7111c9c0c06990327b9127058d67' name=secret>
+                <button type=submit class=button>Secret Submit</button>
+            </form>
+        </section>
+        """
+        crypto_block = f"""
+        <section class=blog-header>
+            <h1>Access Denied!</h1>
+            <hr>
+        </section>
+        """
+        if "secret=" in qs:
+            value = qs.split("=")[1]
+            if value:
+                return Response(crypto_block, status=200)
+
+        return Response(parameter_block, status=200)
+
+    async def setup_after_prep(self, module_test):
+        module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA"
+        expect_args = re.compile("/")
+        module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)
+
+    def check(self, module_test, events):
+        cryptoerror_parameter_extracted = False
+        cryptoerror_finding_emitted = False
+
+        for e in events:
+            print(e)
+            print(e.type)
+
+            if e.type == "WEB_PARAMETER":
+                if "HTTP Extracted Parameter [secret] (GET Form Submodule)" in e.data["description"]:
+                    cryptoerror_parameter_extracted = True
+            if e.type == "FINDING":
+                if (
+                    "Possible Cryptographic Error. Parameter: [secret] Parameter Type: [GETPARAM] Original Value: [08a5a2cea9c5a5576e6e5314edcba581d21c7111c9c0c06990327b9127058d67]"
+                    in e.data["description"]
+                ):
+                    cryptoerror_finding_emitted = True
+        assert cryptoerror_parameter_extracted, "Parameter not extracted"
+        assert cryptoerror_finding_emitted, "Crypto Error Message FINDING not emitted"
+
+
+class Test_Lightfuzz_crypto_error_falsepositive(ModuleTestBase):
+
+    targets = ["http://127.0.0.1:8888/"]
+    modules_overrides = ["httpx", "excavate", "lightfuzz"]
+    config_overrides = {
+        "interactsh_disable": True,
+        "modules": {
+            "lightfuzz": {"enabled_submodules": ["crypto"]},
+        },
+    }
+
+    def request_handler(self, request):
+        fp_block = """
+        <section class=secret>
+            <form action=/ method=GET>
+                <input type=text value='08a5a2cea9c5a5576e6e5314edcba581d21c7111c9c0c06990327b9127058d67' name=secret>
+                <button type=submit class=button>Secret Submit</button>
+            </form>
+            <h1>Access Denied!</h1>
+        </section>
+        """
+        return Response(fp_block, status=200)
+
+    async def setup_after_prep(self, module_test):
+        module_test.scan.modules["lightfuzz"].helpers.rand_string = lambda *args, **kwargs: "AAAAAAAAAAAAAA"
+        expect_args = re.compile("/")
+        module_test.set_expect_requests_handler(expect_args=expect_args, request_handler=self.request_handler)
+
+    def check(self, module_test, events):
+        cryptoerror_parameter_extracted = False
+        cryptoerror_finding_emitted = False
+
+        for e in events:
+            if e.type == "WEB_PARAMETER":
+                if "HTTP Extracted Parameter [secret] (GET Form Submodule)" in e.data["description"]:
+                    cryptoerror_parameter_extracted = True
+            if e.type == "FINDING":
+                if "Possible Cryptographic Error" in e.data["description"]:
+                    cryptoerror_finding_emitted = True
+        assert cryptoerror_parameter_extracted, "Parameter not extracted"
+        assert (
+            not cryptoerror_finding_emitted
+        ), "Crypto Error Message FINDING was emitted (it is an intentional false positive)"

From 5f126f976806a000b1467c774262371e4cae42ce Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 19:23:23 -0400
Subject: [PATCH 02/12] flake8

---
 bbot/core/event/base.py                                    | 7 -------
 bbot/core/helpers/misc.py                                  | 1 -
 bbot/modules/internal/excavate.py                          | 2 --
 bbot/modules/lightfuzz_submodules/base.py                  | 2 --
 bbot/modules/lightfuzz_submodules/sqli.py                  | 2 --
 .../test/test_step_2/module_tests/test_module_lightfuzz.py | 6 +++---
 6 files changed, 3 insertions(+), 17 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index b096aa0957..a50fe0278b 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -5,7 +5,6 @@
 import uuid
 import json
 import base64
-import inspect
 import asyncio
 import logging
 import tarfile
@@ -1393,10 +1392,6 @@ def _build_tree(element, subdict):
 
             return ET.tostring(root_element, encoding="utf-8").decode("utf-8")
 
-        @staticmethod
-        def is_ascii_printable(s):
-            return all(32 <= ord(char) < 127 for char in s)
-
         preprocess_map = {
             "base64": preprocess_base64,
             "hex": preprocess_hex,
@@ -1675,8 +1670,6 @@ def _data_id(self):
         return f"{url}:{name}:{param_type}:{subparameter}"
 
     def _outgoing_dedup_hash(self, event):
-
-        envelopes = self.data.get("envelopes")
         return hash(
             (
                 str(event.host),
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 6cb3bf9138..6c8733eed6 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -3,7 +3,6 @@
 import copy
 import json
 import math
-import base64
 import random
 import string
 import asyncio
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 0314986306..bf8aea9088 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -450,11 +450,9 @@ def extract(self):
                         form_action = form_action.lstrip(".")
 
                     form_parameters = {}
-                    match_found = False
                     for form_content_regex_name, form_content_regex in self.form_content_regexes.items():
                         input_tags = form_content_regex.findall(form_content)
                         if input_tags:
-                            match_found = True
 
                             if form_content_regex_name == "input_tag_novalue_regex":
                                 form_parameters[input_tags[0]] = None
diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py
index 31438592be..b1622270c3 100644
--- a/bbot/modules/lightfuzz_submodules/base.py
+++ b/bbot/modules/lightfuzz_submodules/base.py
@@ -1,6 +1,4 @@
 import copy
-import base64
-
 
 class BaseLightfuzz:
     def __init__(self, lightfuzz, event):
diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py
index ac660fc4cb..cf10aff9d1 100644
--- a/bbot/modules/lightfuzz_submodules/sqli.py
+++ b/bbot/modules/lightfuzz_submodules/sqli.py
@@ -1,10 +1,8 @@
 from .base import BaseLightfuzz
 from bbot.errors import HttpCompareError
 
-import urllib.parse
 import statistics
 
-
 class SQLiLightfuzz(BaseLightfuzz):
     expected_delay = 5
 
diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
index eb5607085e..cdbe9ef4ce 100644
--- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
+++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
@@ -320,7 +320,7 @@ def request_handler(self, request):
                 else:
                     extracted_value = "[Parameter not found in JSON]"
 
-            except (json.JSONDecodeError, ValueError) as e:
+            except (json.JSONDecodeError, ValueError):
                 extracted_value = "[Invalid hex or JSON format]"
 
             xss_block = f"""
@@ -372,7 +372,7 @@ def request_handler(self, request):
                 else:
                     extracted_value = "[Parameter not found in JSON]"
 
-            except (json.JSONDecodeError, base64.binascii.Error) as e:
+            except (json.JSONDecodeError, base64.binascii.Error):
                 extracted_value = "[Invalid base64 or JSON format]"
 
             xss_block = f"""
@@ -425,7 +425,7 @@ def request_handler(self, request):
                 else:
                     extracted_value = "[Parameter not found in XML]"
 
-            except (ET.ParseError, base64.binascii.Error) as e:
+            except (ET.ParseError, base64.binascii.Error):
                 extracted_value = "[Invalid base64 or XML format]"
 
             xss_block = f"""

From b1fabdd311ab8b7a63547d691cdc8d8fd4b372e2 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 19:27:08 -0400
Subject: [PATCH 03/12] black

---
 bbot/modules/lightfuzz_submodules/base.py | 1 +
 bbot/modules/lightfuzz_submodules/sqli.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py
index b1622270c3..51e93cc50b 100644
--- a/bbot/modules/lightfuzz_submodules/base.py
+++ b/bbot/modules/lightfuzz_submodules/base.py
@@ -1,5 +1,6 @@
 import copy
 
+
 class BaseLightfuzz:
     def __init__(self, lightfuzz, event):
         self.lightfuzz = lightfuzz
diff --git a/bbot/modules/lightfuzz_submodules/sqli.py b/bbot/modules/lightfuzz_submodules/sqli.py
index cf10aff9d1..0cf17975ef 100644
--- a/bbot/modules/lightfuzz_submodules/sqli.py
+++ b/bbot/modules/lightfuzz_submodules/sqli.py
@@ -3,6 +3,7 @@
 
 import statistics
 
+
 class SQLiLightfuzz(BaseLightfuzz):
     expected_delay = 5
 

From 2f6062495a33156258e27342ece409da3d05437e Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 19:29:36 -0400
Subject: [PATCH 04/12] clean up debug

---
 bbot/modules/lightfuzz_submodules/base.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bbot/modules/lightfuzz_submodules/base.py b/bbot/modules/lightfuzz_submodules/base.py
index 51e93cc50b..05d1dbbe0d 100644
--- a/bbot/modules/lightfuzz_submodules/base.py
+++ b/bbot/modules/lightfuzz_submodules/base.py
@@ -22,7 +22,7 @@ async def send_probe(self, probe):
         probe = self.probe_value_outgoing(probe)
         getparams = {self.event.data["name"]: probe}
         url = self.lightfuzz.helpers.add_get_params(self.event.data["url"], getparams, encode=False).geturl()
-        self.lightfuzz.critical(f"lightfuzz sending probe with URL: {url}")
+        self.lightfuzz.debug(f"lightfuzz sending probe with URL: {url}")
         r = await self.lightfuzz.helpers.request(method="GET", url=url, allow_redirects=False, retries=2, timeout=10)
         if r:
             return r.text
@@ -193,7 +193,6 @@ def probe_value_incoming(self, populate_empty=True):
             probe_value = self.lightfuzz.helpers.rand_string(8, numeric_only=True)
         self.lightfuzz.debug(f"probe_value_incoming (before modification): {probe_value}")
         envelopes_instance = getattr(self.event, "envelopes", None)
-        self.lightfuzz.hugesuccess(envelopes_instance)
         probe_value = envelopes_instance.remove_envelopes(probe_value)
         self.lightfuzz.debug(f"probe_value_incoming (after modification): {probe_value}")
         return probe_value
@@ -201,7 +200,6 @@ def probe_value_incoming(self, populate_empty=True):
     def probe_value_outgoing(self, outgoing_probe_value):
         self.lightfuzz.debug(f"probe_value_outgoing (before modification): {outgoing_probe_value}")
         envelopes_instance = getattr(self.event, "envelopes", None)
-        self.lightfuzz.hugesuccess(envelopes_instance)
         outgoing_probe_value = envelopes_instance.add_envelopes(outgoing_probe_value)
         self.lightfuzz.debug(f"probe_value_outgoing (after modification): {outgoing_probe_value}")
         return outgoing_probe_value

From 07eccb5a3ee1a2c52df85198cd400e5e70e40051 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 19:33:54 -0400
Subject: [PATCH 05/12] remove debug

---
 .../test_step_2/module_tests/test_module_lightfuzz.py     | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
index cdbe9ef4ce..e460d2d2bf 100644
--- a/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
+++ b/bbot/test/test_step_2/module_tests/test_module_lightfuzz.py
@@ -262,8 +262,6 @@ def request_handler(self, request):
             <hr>
         </section>
         """
-            print("XSS BLOCK:")
-            print(xss_block)
             return Response(xss_block, status=200)
         return Response(parameter_block, status=200)
 
@@ -329,8 +327,6 @@ def request_handler(self, request):
             <hr>
         </section>
         """
-            print("XSS BLOCK:")
-            print(xss_block)
             return Response(xss_block, status=200)
 
         return Response(parameter_block, status=200)
@@ -381,8 +377,6 @@ def request_handler(self, request):
             <hr>
         </section>
         """
-            print("XSS BLOCK:")
-            print(xss_block)
             return Response(xss_block, status=200)
 
         return Response(parameter_block, status=200)
@@ -434,8 +428,6 @@ def request_handler(self, request):
             <hr>
         </section>
         """
-            print("XSS BLOCK:")
-            print(xss_block)
             return Response(xss_block, status=200)
 
         return Response(parameter_block, status=200)

From 9da63bb88c96aace417f77594ea06a6b0285acf9 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 21:02:07 -0400
Subject: [PATCH 06/12] replacing dummy

---
 bbot/core/event/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index a50fe0278b..82528fb886 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -2057,7 +2057,6 @@ def make_event(
             _internal=internal,
         )
 
-
 def event_from_json(j, siem_friendly=False):
     """
     Creates an event object from a JSON dictionary.
@@ -2088,6 +2087,7 @@ def event_from_json(j, siem_friendly=False):
             "tags": j.get("tags", []),
             "confidence": j.get("confidence", 100),
             "context": j.get("discovery_context", None),
+            "dummy": True,
         }
         if siem_friendly:
             data = j["data"][event_type]

From 0d395f923119351cf9cd8ebe14c3f3e585d4592f Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 21:03:59 -0400
Subject: [PATCH 07/12] black

---
 bbot/core/event/base.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 82528fb886..40cae72024 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -2057,6 +2057,7 @@ def make_event(
             _internal=internal,
         )
 
+
 def event_from_json(j, siem_friendly=False):
     """
     Creates an event object from a JSON dictionary.

From d66badd14cad63bddd2f78ce93988be0952abd05 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 23:38:32 -0400
Subject: [PATCH 08/12] tweak debug msg

---
 bbot/core/event/base.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 40cae72024..ad4a3006f0 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1466,7 +1466,7 @@ def recurse_envelopes(self, value, envelopes=None, end_format=None):
             if envelopes is None:
                 envelopes = []
             log.debug(
-                f"Starting recurse with value: {value}, current envelopes: {', '.join(envelopes)}, current end format: {end_format}"
+                f"Starting envelope recurse with value: [{value}], current envelopes: [{', '.join(envelopes)}], current end format: {end_format}"
             )
 
             if value is None or value == "" or isinstance(value, int):
@@ -2102,7 +2102,6 @@ def event_from_json(j, siem_friendly=False):
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
-
         event.timestamp = datetime.datetime.fromisoformat(j["timestamp"])
         event.scope_distance = j["scope_distance"]
         parent_id = j.get("parent", None)

From 8ec46cbb93e39454dbc393ec8f02812c1be17421 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 23:39:07 -0400
Subject: [PATCH 09/12] fixing path reconstruction bug

---
 bbot/modules/internal/excavate.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index bf8aea9088..ded56bd770 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -329,6 +329,8 @@ class excavateTestRule(ExcavateRule):
             "ASP.NET_SessionId",
             "JSESSIONID",
             "PHPSESSID",
+            "AWSALB",
+            "AWSALBCORS",
         ]
     )
 
@@ -524,13 +526,19 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte
                             self.excavate.debug(
                                 f"Found Parameter [{parameter_name}] in [{parameterExtractorSubModule.name}] ParameterExtractor Submodule"
                             )
-                            endpoint = event.data["url"] if not endpoint else endpoint
-                            path = f"{event.parsed_url.path.lstrip('/')}"
-                            url = (
-                                endpoint
-                                if endpoint.startswith(("http://", "https://"))
-                                else f"{event.parsed_url.scheme}://{event.parsed_url.netloc}{path}{endpoint}"
-                            )
+                            # If we have a full URL, leave it as-is
+                            if not endpoint.startswith(("http://", "https://")):
+
+                                # The endpoint is usually a form action - we should use it if we have it. If not, defautl to URL.
+                                path = event.parsed_url.path if not endpoint else endpoint
+                                # Normalize path by remove leading slash
+                                path = path.lstrip("/")
+
+                                # Ensure the base URL has a single slash between path and endpoint
+                                url = f"{event.parsed_url.scheme}://{event.parsed_url.netloc}/{path}"
+                            else:
+                                url = endpoint
+
                             if self.excavate.helpers.validate_parameter(parameter_name, parameter_type):
 
                                 if self.excavate.in_bl(parameter_name) == False:

From 45c6c1b96e4dd28c6de62a808f1519428db82e8d Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Tue, 8 Oct 2024 23:42:41 -0400
Subject: [PATCH 10/12] adding additional test

---
 bbot/modules/lightfuzz_submodules/crypto.py   |  2 +-
 .../module_tests/test_module_excavate.py      | 23 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/lightfuzz_submodules/crypto.py b/bbot/modules/lightfuzz_submodules/crypto.py
index da7e52a63c..526877339c 100644
--- a/bbot/modules/lightfuzz_submodules/crypto.py
+++ b/bbot/modules/lightfuzz_submodules/crypto.py
@@ -246,7 +246,7 @@ async def fuzz(self):
 
         baseline_probe = await self.baseline_probe(cookies)
         if not baseline_probe:
-            self.lightfuzz.critical(f"Couldn't get baseline_probe for url {self.event.data['url']}, aborting")
+            self.lightfuzz.warning(f"Couldn't get baseline_probe for url {self.event.data['url']}, aborting")
             return
 
         try:
diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py
index 551dd3b031..814c634931 100644
--- a/bbot/test/test_step_2/module_tests/test_module_excavate.py
+++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py
@@ -582,6 +582,29 @@ def check(self, module_test, events):
         assert excavate_getparam_extraction, "Excavate failed to extract web parameter"
 
 
+class TestExcavateParameterExtraction_getparam_novalue(TestExcavateParameterExtraction_getparam):
+    getparam_extract_html = """
+                   <section class=search>
+                        <form action="/catalog" method=GET>
+                            <input type=text id="searchBar" placeholder="Search products" name="searchTerm">
+                            <script>
+                                var searchText = '';
+                                document.getElementById('searchBar').value = searchText;
+                            </script>
+                            <button type=submit class=button>Search</button>
+                        </form>
+                    </section>
+    """
+
+    def check(self, module_test, events):
+        excavate_getparam_extraction = False
+        for e in events:
+            if e.type == "WEB_PARAMETER":
+                if "HTTP Extracted Parameter [searchTerm] (GET Form Submodule)" in e.data["description"]:
+                    excavate_getparam_extraction = True
+        assert excavate_getparam_extraction, "Excavate failed to extract web parameter"
+
+
 class TestExcavateParameterExtraction_json(ModuleTestBase):
     targets = ["http://127.0.0.1:8888/"]
     modules_overrides = ["httpx", "excavate", "paramminer_getparams"]

From 069fdc64447ca1c8326d29cb5ec6960b1d633db3 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Wed, 9 Oct 2024 07:49:45 -0400
Subject: [PATCH 11/12] 3.9 compatibility hack

---
 bbot/core/event/base.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ad4a3006f0..ea7ca2aad4 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1436,13 +1436,15 @@ def remove_envelopes(self, value):
             for env in self.envelopes:
                 func = self.preprocess_map.get(env)
                 if func:
+                    # python3.9 compatibility hack
+                    if isinstance(func, staticmethod):
+                        func = func.__get__(None, self.__class__)  # Unwrap staticmethod
                     value = func(value)
 
             # Dynamically select the appropriate isolate function based on the final format
             isolate_func = self.format_isolate_map.get(self.end_format_type)
             if isolate_func:
                 return isolate_func(self)
-
             return value
 
         def add_envelopes(self, value):
@@ -1453,12 +1455,18 @@ def add_envelopes(self, value):
             # Dynamically select the appropriate update function based on the final format
             update_func = self.format_update_map.get(self.end_format_type)
             if update_func:
+                # python3.9 compatibility hack
+                if isinstance(update_func, staticmethod):
+                    update_func = update_func.__get__(None, self.__class__)
                 value = update_func(self, value)
 
             # Apply the envelopes in reverse order
             for env in self.envelopes[::-1]:
                 func = self.postprocess_map.get(env)
                 if func:
+                    # python3.9 compatibility hack
+                    if isinstance(func, staticmethod):
+                        func = func.__get__(None, self.__class__)
                     value = func(value)
             return value
 

From 9cd281a3cb6698acfc735841b538e159400586a9 Mon Sep 17 00:00:00 2001
From: liquidsec <paul.mueller08@gmail.com>
Date: Wed, 9 Oct 2024 13:41:41 -0400
Subject: [PATCH 12/12] removing unnecessary threading lock handling

---
 bbot/core/event/base.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index ea7ca2aad4..e8429869fc 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -1,7 +1,6 @@
 from collections import defaultdict
 import io
 import re
-import threading
 import uuid
 import json
 import base64
@@ -630,14 +629,10 @@ def clone(self):
 
         # Handle attributes that need deep copying manually
         setattr(cloned_event, "envelopes", deepcopy(self.envelopes))
-        #   cloned_event.scan = deepcopy(self.scan)
 
         # Re-assign a new UUID
         cloned_event.uuid = uuid.uuid4()
 
-        # Re-create the unpickleable lock object
-        cloned_event.lock = threading.RLock()
-
         return cloned_event
 
     def _host(self):