diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py
index 96c4594e3..92905bd01 100644
--- a/bbot/core/event/base.py
+++ b/bbot/core/event/base.py
@@ -90,6 +90,8 @@ class BaseEvent:
_always_emit = False
# Always emit events with these tags even if they're not in scope
_always_emit_tags = ["affiliate"]
+ # Whether this event has been retroactively marked as part of an important discovery chain
+ _graph_important = False
# Exclude from output modules
_omit = False
# Disables certain data validations
@@ -143,9 +145,6 @@ def __init__(
self._module_priority = None
self._resolved_hosts = set()
- self._made_internal = False
- # whether to force-send to output modules
- self._force_output = False
# keep track of whether this event has been recorded by the scan
self._stats_recorded = False
@@ -199,7 +198,7 @@ def __init__(
if not self._dummy:
# removed this second part because it was making certain sslcert events internal
if _internal: # or source._internal:
- self.make_internal()
+ self.internal = True
# an event indicating whether the event has undergone DNS resolution
self._resolved = asyncio.Event()
@@ -224,6 +223,34 @@ def data(self, data):
self._port = None
self._data = data
+ @property
+ def internal(self):
+ return self._internal
+
+ @internal.setter
+ def internal(self, value):
+ """
+ Marks the event as internal, excluding it from output but allowing normal exchange between scan modules.
+
+ Internal events are typically speculative and may not be interesting by themselves but can lead to
+ the discovery of interesting events. This method sets the `_internal` attribute to True and adds the
+ "internal" tag.
+
+ Examples of internal events include `OPEN_TCP_PORT`s from the `speculate` module,
+ `IP_ADDRESS`es from the `ipneighbor` module, or out-of-scope `DNS_NAME`s that originate
+ from DNS resolutions.
+
+ The purpose of internal events is to enable speculative/explorative discovery without cluttering
+ the console with irrelevant or uninteresting events.
+ """
+ if not value in (True, False):
+ raise ValueError(f'"internal" must be boolean, not {type(value)}')
+ if value == True:
+ self.add_tag("internal")
+ else:
+ self.remove_tag("internal")
+ self._internal = value
+
@property
def host(self):
"""
@@ -292,7 +319,9 @@ def remove_tag(self, tag):
@property
def always_emit(self):
- return self._always_emit or any(t in self.tags for t in self._always_emit_tags)
+ always_emit_tags = any(t in self.tags for t in self._always_emit_tags)
+ no_host_information = not bool(self.host)
+ return self._always_emit or always_emit_tags or no_host_information
@property
def id(self):
@@ -327,11 +356,21 @@ def scope_distance(self, scope_distance):
else:
new_scope_distance = min(self.scope_distance, scope_distance)
if self._scope_distance != new_scope_distance:
- self._scope_distance = new_scope_distance
+ # remove old scope distance tags
for t in list(self.tags):
if t.startswith("distance-"):
self.remove_tag(t)
- self.add_tag(f"distance-{new_scope_distance}")
+ if scope_distance == 0:
+ self.add_tag("in-scope")
+ self.remove_tag("affiliate")
+ else:
+ self.remove_tag("in-scope")
+ self.add_tag(f"distance-{new_scope_distance}")
+ self._scope_distance = new_scope_distance
+ # apply recursively to parent events
+ source_scope_distance = getattr(self.source, "scope_distance", -1)
+ if source_scope_distance >= 0 and self != self.source:
+ self.source.scope_distance = scope_distance + 1
@property
def source(self):
@@ -396,95 +435,6 @@ def get_sources(self, omit=False):
e = source
return sources
- def make_internal(self):
- """
- Marks the event as internal, excluding it from output but allowing normal exchange between scan modules.
-
- Internal events are typically speculative and may not be interesting by themselves but can lead to
- the discovery of interesting events. This method sets the `_internal` attribute to True, adds the
- "internal" tag, and ensures the event is marked as made internal (useful for later reversion).
-
- Examples of internal events include `OPEN_TCP_PORT`s from the `speculate` module,
- `IP_ADDRESS`es from the `ipneighbor` module, or out-of-scope `DNS_NAME`s that originate
- from DNS resolutions.
-
- Once an event is marked as internal, all of its future children become internal as well.
- If `ScanManager._emit_event()` determines the event is interesting, it may be reverted back to its
- original state and forcefully re-emitted along with the whole chain of internal events.
-
- The purpose of internal events is to enable speculative/explorative discovery without cluttering
- the console with irrelevant or uninteresting events.
- """
- if not self._made_internal:
- self._internal = True
- self.add_tag("internal")
- self._made_internal = True
-
- def unmake_internal(self, set_scope_distance=None, force_output=False):
- """
- Reverts the event from being internal, optionally forcing it to be included in output and setting its scope distance.
-
- Removes the 'internal' tag, resets the `_internal` attribute, and adjusts scope distance if specified.
- Optionally, forces the event to be included in the output. Also, if any source events are internal, they
- are also reverted recursively.
-
- This typically happens in `ScanManager._emit_event()` if the event is determined to be interesting.
-
- Parameters:
- set_scope_distance (int, optional): If specified, sets the scope distance to this value.
- force_output (bool or str, optional): If True, forces the event to be included in output.
- If set to "trail_only", only its source events are modified.
-
- Returns:
- list: A list of source events that were also reverted from being internal.
- """
- source_trail = []
- self.remove_tag("internal")
- if self._made_internal:
- if set_scope_distance is not None:
- self.scope_distance = set_scope_distance
- self._internal = False
- self._made_internal = False
- if force_output is True:
- self._force_output = True
- if force_output == "trail_only":
- force_output = True
-
- # if our source event is internal, unmake it too
- if getattr(self.source, "_internal", False):
- source_scope_distance = None
- if set_scope_distance is not None:
- source_scope_distance = set_scope_distance + 1
- source_trail += self.source.unmake_internal(
- set_scope_distance=source_scope_distance, force_output=force_output
- )
- source_trail.append(self.source)
-
- return source_trail
-
- def set_scope_distance(self, d=0):
- """
- Sets the scope distance for the event and its parent events, while considering module-specific scoping rules.
-
- Unmakes the event internal if needed and adjusts its scope distance. If the distance is set to 0,
- adds the 'in-scope' tag to the event. Takes into account module-specific scoping preferences unless
- the event type is "DNS_NAME".
-
- Parameters:
- d (int): The scope distance to set for this event.
-
- Returns:
- list: A list of parent events whose scope distance was also set.
- """
- source_trail = []
- # keep the event internal if the module requests so, unless it's a DNS_NAME
- if getattr(self.module, "_scope_shepherding", True) or self.type in ("DNS_NAME",):
- source_trail = self.unmake_internal(set_scope_distance=d, force_output="trail_only")
- self.scope_distance = d
- if d == 0:
- self.add_tag("in-scope")
- return source_trail
-
def _host(self):
return ""
@@ -1240,8 +1190,8 @@ def make_event(
data.module = module
if source is not None:
data.source = source
- if internal == True and not data._made_internal:
- data.make_internal()
+ if internal == True:
+ data.internal = True
event_type = data.type
return data
else:
diff --git a/bbot/core/helpers/dns.py b/bbot/core/helpers/dns.py
index 860853661..58dbce3e2 100644
--- a/bbot/core/helpers/dns.py
+++ b/bbot/core/helpers/dns.py
@@ -1,3 +1,4 @@
+import dns
import time
import asyncio
import logging
@@ -89,7 +90,7 @@ def __init__(self, parent_helper):
self.timeout = self.parent_helper.config.get("dns_timeout", 5)
self.retries = self.parent_helper.config.get("dns_retries", 1)
self.abort_threshold = self.parent_helper.config.get("dns_abort_threshold", 50)
- self.max_dns_resolve_distance = self.parent_helper.config.get("max_dns_resolve_distance", 4)
+ self.max_dns_resolve_distance = self.parent_helper.config.get("max_dns_resolve_distance", 5)
self.resolver.timeout = self.timeout
self.resolver.lifetime = self.timeout
self._resolver_list = None
@@ -132,6 +133,10 @@ def __init__(self, parent_helper):
self._event_cache = self.parent_helper.CacheDict(max_size=10000)
self._event_cache_locks = NamedLock()
+ # for mocking DNS queries
+ self._orig_resolve_raw = None
+ self._mock_table = {}
+
# copy the system's current resolvers to a text file for tool use
self.system_resolvers = dns.resolver.Resolver().nameservers
self.resolver_file = self.parent_helper.tempfile(self.system_resolvers, pipe=False)
@@ -220,13 +225,7 @@ async def resolve_raw(self, query, **kwargs):
kwargs.pop("rdtype", None)
if "type" in kwargs:
t = kwargs.pop("type")
- if isinstance(t, str):
- if t.strip().lower() in ("any", "all", "*"):
- types = self.all_rdtypes
- else:
- types = [t.strip().upper()]
- elif any([isinstance(t, x) for x in (list, tuple)]):
- types = [str(_).strip().upper() for _ in t]
+ types = self._parse_rdtype(t, default=types)
for t in types:
r, e = await self._resolve_hostname(query, rdtype=t, **kwargs)
if r:
@@ -500,7 +499,7 @@ async def resolve_event(self, event, minimal=False):
event_blacklisted = False
try:
- if not event.host or event.type in ("IP_RANGE",):
+ if (not event.host) or (event.type in ("IP_RANGE",)):
return event_tags, event_whitelisted, event_blacklisted, dns_children
# lock to ensure resolution of the same host doesn't start while we're working here
@@ -1016,6 +1015,16 @@ async def _connectivity_check(self, interval=5):
self._errors.clear()
return False
+ def _parse_rdtype(self, t, default=None):
+ if isinstance(t, str):
+ if t.strip().lower() in ("any", "all", "*"):
+ return self.all_rdtypes
+ else:
+ return [t.strip().upper()]
+ elif any([isinstance(t, x) for x in (list, tuple)]):
+ return [str(_).strip().upper() for _ in t]
+ return default
+
def debug(self, *args, **kwargs):
if self._debug:
log.debug(*args, **kwargs)
@@ -1027,3 +1036,28 @@ def _get_dummy_module(self, name):
dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS")
self._dummy_modules[name] = dummy_module
return dummy_module
+
+ def mock_dns(self, dns_dict):
+ if self._orig_resolve_raw is None:
+ self._orig_resolve_raw = self.resolve_raw
+
+ async def mock_resolve_raw(query, **kwargs):
+ results = []
+ errors = []
+ types = self._parse_rdtype(kwargs.get("type", ["A", "AAAA"]))
+ for t in types:
+ with suppress(KeyError):
+ results += self._mock_table[(query, t)]
+ return results, errors
+
+ for (query, rdtype), answers in dns_dict.items():
+ if isinstance(answers, str):
+ answers = [answers]
+ for answer in answers:
+ rdata = dns.rdata.from_text("IN", rdtype, answer)
+ try:
+ self._mock_table[(query, rdtype)].append((rdtype, rdata))
+ except KeyError:
+ self._mock_table[(query, rdtype)] = [(rdtype, [rdata])]
+
+ self.resolve_raw = mock_resolve_raw
diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 125b9c14b..6dee96745 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -61,7 +61,7 @@ class BaseModule:
failed_request_abort_threshold (int): Threshold for setting error state after failed HTTP requests (only takes effect when `request_with_fail_count()` is used. Default is 5.
- _scope_shepherding (bool): When set to false, prevents events generated by this module from being automatically marked as in-scope. Default is True. Useful for low-confidence modules like speculate and ipneighbor.
+ _preserve_graph (bool): When set to True, accept events that may be duplicates but are necessary for construction of complete graph. Typically only enabled for output modules that need to maintain full chains of events, e.g. `neo4j` and `json`. Default is False.
_stats_exclude (bool): Whether to exclude this module from scan statistics. Default is False.
@@ -99,7 +99,7 @@ class BaseModule:
batch_wait = 10
failed_request_abort_threshold = 5
- _scope_shepherding = True
+ _preserve_graph = False
_stats_exclude = False
_qsize = 0
_priority = 3
@@ -122,6 +122,8 @@ def __init__(self, scan):
self._log = None
self._incoming_event_queue = None
self._outgoing_event_queue = None
+ # track incoming events to prevent unwanted duplicates
+ self._incoming_dup_tracker = set()
# seconds since we've submitted a batch
self._last_submitted_batch = None
# additional callbacks to be executed alongside self.cleanup()
@@ -674,9 +676,23 @@ def _event_precheck(self, event):
# then skip the event.
# this helps avoid double-portscanning both an individual IP and its parent CIDR.
return False, "module consumes IP ranges directly"
+
return True, "precheck succeeded"
async def _event_postcheck(self, event):
+ """
+ A simple wrapper for dup tracking
+ """
+ acceptable, reason = await self.__event_postcheck(event)
+ if acceptable:
+ # check duplicates
+ is_incoming_duplicate = self.is_incoming_duplicate(event, add=True)
+ if is_incoming_duplicate and not self.accept_dupes:
+ return False, f"module has already seen {event}"
+
+ return acceptable, reason
+
+ async def __event_postcheck(self, event):
"""
Post-checks an event to determine if it should be accepted by the module for handling.
@@ -689,14 +705,6 @@ async def _event_postcheck(self, event):
Returns:
tuple: A tuple (bool, str) where the bool indicates if the event should be accepted, and the str gives the reason.
- Examples:
- >>> async def custom_filter(event):
- ... if event.data not in ["evilcorp.com"]:
- ... return False, "it's not on the cool list"
- ...
- >>> self.filter_event = custom_filter
- >>> result, reason = await self._event_postcheck(event)
-
Notes:
- Override the `filter_event` method for custom filtering logic.
- This method also maintains host-based tracking when the `per_host_only` flag is set.
@@ -706,8 +714,13 @@ async def _event_postcheck(self, event):
if event.type in ("FINISHED",):
return True, ""
- # reject out-of-scope events for active modules
- # TODO: reconsider this
+ # force-output certain events to the graph
+ if self._is_graph_important(event):
+ return True, "event is critical to the graph"
+
+ # don't send out-of-scope targets to active modules
+ # this only takes effect if your target and whitelist are different
+ # TODO: the logic here seems incomplete, it could probably use some work.
if "active" in self.flags and "target" in event.tags and event not in self.scan.whitelist:
return False, "it is not in whitelist and module has active flag"
@@ -770,7 +783,7 @@ async def _cleanup(self):
async with self.scan._acatch(context), self._task_counter.count(context):
await self.helpers.execute_sync_or_async(callback)
- async def queue_event(self, event):
+ async def queue_event(self, event, precheck=True):
"""
Asynchronously queues an incoming event to the module's event queue for further processing.
@@ -793,7 +806,9 @@ async def queue_event(self, event):
if self.incoming_event_queue is False:
self.debug(f"Not in an acceptable state to queue incoming event")
return
- acceptable, reason = self._event_precheck(event)
+ acceptable, reason = True, "precheck was skipped"
+ if precheck:
+ acceptable, reason = self._event_precheck(event)
if not acceptable:
if reason and reason != "its type is not in watched_events":
self.debug(f"Not accepting {event} because {reason}")
@@ -876,6 +891,29 @@ def set_error_state(self, message=None, clear_outgoing_queue=False):
while 1:
self.outgoing_event_queue.get_nowait()
+ def is_incoming_duplicate(self, event, add=False):
+ event_hash = self._incoming_dedup_hash(event)
+ is_dup = event_hash in self._incoming_dup_tracker
+ if add:
+ self._incoming_dup_tracker.add(event_hash)
+ return is_dup
+
+ def _incoming_dedup_hash(self, event):
+ """
+ Determines the criteria for what is considered to be a duplicate event if `accept_dupes` is False.
+ """
+ if self.per_host_only:
+ return self.get_per_host_hash(event)
+ elif self.per_domain_only:
+ return self.get_per_domain_hash(event)
+ return hash(event)
+
+ def _outgoing_dedup_hash(self, event):
+ """
+ Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
+ """
+ return hash(event)
+
def get_per_host_hash(self, event):
"""
Computes a per-host hash value for a given event. This method may be optionally overridden in subclasses.
@@ -1116,6 +1154,9 @@ def log_table(self, *args, **kwargs):
self.verbose(f"Wrote {table_name} to {filename}")
return table
+ def _is_graph_important(self, event):
+ return self._preserve_graph and getattr(event, "_graph_important", False)
+
def stdout(self, *args, **kwargs):
"""Writes log messages directly to standard output.
diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py
index 6341f16ea..3c78f9506 100644
--- a/bbot/modules/httpx.py
+++ b/bbot/modules/httpx.py
@@ -153,7 +153,7 @@ async def handle_batch(self, *events):
else:
url_event._resolved.set()
# HTTP response
- self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags, internal=True)
+ self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags)
async def cleanup(self):
resume_file = self.helpers.current_dir / "resume.cfg"
diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py
index 4aa3bb616..0a506393b 100644
--- a/bbot/modules/internal/speculate.py
+++ b/bbot/modules/internal/speculate.py
@@ -1,7 +1,6 @@
import random
import ipaddress
-from bbot.core.helpers.misc import parse_port_string
from bbot.modules.internal.base import BaseInternalModule
@@ -31,7 +30,6 @@ class speculate(BaseInternalModule):
"ports": "The set of ports to speculate on",
}
scope_distance_modifier = 1
- _scope_shepherding = False
_priority = 4
async def setup(self):
@@ -43,7 +41,7 @@ async def setup(self):
port_string = self.config.get("ports", "80,443")
try:
- self.ports = parse_port_string(port_string)
+ self.ports = self.helpers.parse_port_string(str(port_string))
except ValueError as e:
self.warning(f"Error parsing ports: {e}")
return False
diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py
index 1ce8cf0f9..b6688abee 100644
--- a/bbot/modules/ipneighbor.py
+++ b/bbot/modules/ipneighbor.py
@@ -11,7 +11,6 @@ class ipneighbor(BaseModule):
options = {"num_bits": 4}
options_desc = {"num_bits": "Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)"}
scope_distance_modifier = 1
- _scope_shepherding = False
async def setup(self):
self.processed = set()
diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py
index 204a8f44a..99a03b0cf 100644
--- a/bbot/modules/output/base.py
+++ b/bbot/modules/output/base.py
@@ -24,15 +24,16 @@ def _event_precheck(self, event):
# exclude certain URLs (e.g. javascript):
if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
return False, "its extension was listed in url_extension_httpx_only"
- # forced events like intermediary links in a DNS resolution chain
# output module specific stuff
# omitted events such as HTTP_RESPONSE etc.
if event._omit:
return False, "_omit is True"
- # forced events like intermediary links in a DNS resolution chain
- if event._force_output:
- return True, "_force_output is True"
+
+ # force-output certain events to the graph
+ if self._is_graph_important(event):
+ return True, "event is critical to the graph"
+
# internal events like those from speculate, ipneighbor
# or events that are over our report distance
if event._internal:
@@ -53,6 +54,13 @@ def _event_precheck(self, event):
return False, "module consumes IP ranges directly"
return True, "precheck succeeded"
+ def is_incoming_duplicate(self, event, add=False):
+ is_incoming_duplicate = super().is_incoming_duplicate(event, add=add)
+ # make exception for graph-important events
+ if self._is_graph_important(event):
+ return False
+ return is_incoming_duplicate
+
def _prep_output_dir(self, filename):
self.output_file = self.config.get("output_file", "")
if self.output_file:
diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py
index e37a3829d..f13cf7808 100644
--- a/bbot/modules/output/json.py
+++ b/bbot/modules/output/json.py
@@ -9,6 +9,7 @@ class JSON(BaseOutputModule):
meta = {"description": "Output to Newline-Delimited JSON (NDJSON)"}
options = {"output_file": "", "console": False}
options_desc = {"output_file": "Output to file", "console": "Output to console"}
+ _preserve_graph = True
async def setup(self):
self._prep_output_dir("output.ndjson")
diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py
index 2fc05fc9c..4ed01cefb 100644
--- a/bbot/modules/output/neo4j.py
+++ b/bbot/modules/output/neo4j.py
@@ -17,6 +17,7 @@ class neo4j(BaseOutputModule):
}
deps_pip = ["git+https://github.com/blacklanternsecurity/py2neo"]
_batch_size = 50
+ _preserve_graph = True
async def setup(self):
try:
diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py
index 97f3e6388..13d46669d 100644
--- a/bbot/scanner/manager.py
+++ b/bbot/scanner/manager.py
@@ -39,12 +39,23 @@ def __init__(self, scan):
self.scan = scan
- self.incoming_event_queue = asyncio.PriorityQueue()
+ # TODO: consider reworking modules' dedupe policy (accept_dupes)
+ # by creating a function that decides the criteria for what is
+ # considered to be a duplicate (by default this would be a simple
+ # hash(event)), but allowing each module to override it if needed.
+ # If a module used the default function, its dedupe could be done
+ # at the manager level to save memory. If not, it would be done by the scan.
- # tracks duplicate events on a global basis
- self.events_distributed = set()
- # tracks duplicate events on a per-module basis
+ self.incoming_event_queue = asyncio.PriorityQueue()
+ # track incoming duplicates module-by-module (for `suppress_dupes` attribute of modules)
+ self.incoming_dup_tracker = set()
+ # track outgoing duplicates (for `accept_dupes` attribute of modules)
+ self.outgoing_dup_tracker = set()
+ # tracks duplicate events
self.events_accepted = set()
+ # tracks duplicate events for graph purposes
+ # (allows certain duplicates in order to maintain a complete graph)
+ self.events_accepted_graph = set()
self.dns_resolution = self.scan.config.get("dns_resolution", False)
self._task_counter = TaskCounter()
self._new_activity = True
@@ -79,15 +90,19 @@ async def emit_event(self, event, *args, **kwargs):
bbot.scanner: scan._event_thread_pool: running for 0 seconds: ScanManager._emit_event(DNS_NAME("sipfed.online.lync.com"))
"""
async with self._task_counter.count(f"emit_event({event})"):
+ # "quick" queues the event immediately
+ # This is used by speculate
+ quick = kwargs.pop("quick", False)
+
# skip event if it fails precheck
- if not self._event_precheck(event):
- event._resolved.set()
- return
+ if event.type != "DNS_NAME":
+ acceptable = self._event_precheck(event)
+ if not acceptable:
+ event._resolved.set()
+ return
log.debug(f'Module "{event.module}" raised {event}')
- # "quick" queues the event immediately
- quick = kwargs.pop("quick", False)
if quick:
log.debug(f'Module "{event.module}" raised {event}')
event._resolved.set()
@@ -97,11 +112,15 @@ async def emit_event(self, event, *args, **kwargs):
await self.distribute_event(event, *args, **kwargs)
else:
async with self.scan._acatch(context=self._emit_event, finally_callback=event._resolved.set):
- await self._emit_event(event, *args, **kwargs)
+ await self._emit_event(
+ event,
+ *args,
+ **kwargs,
+ )
- def _event_precheck(self, event, exclude=("DNS_NAME",)):
+ def _event_precheck(self, event):
"""
- Check an event previous to its DNS resolution etc. to see if we can save on performance by skipping it
+ Check an event to see if we can skip it to save on performance
"""
if event._dummy:
log.warning(f"Cannot emit dummy event: {event}")
@@ -109,8 +128,10 @@ def _event_precheck(self, event, exclude=("DNS_NAME",)):
if event == event.get_source():
log.debug(f"Skipping event with self as source: {event}")
return False
- if self.is_duplicate_event(event) and not event._force_output:
- log.debug(f"Skipping {event} because it is a duplicate")
+ if event._graph_important:
+ return True
+ if self.is_incoming_duplicate(event, add=True):
+ log.debug(f"Skipping event because it was already emitted by its module: {event}")
return False
return True
@@ -122,9 +143,8 @@ async def _emit_event(self, event, **kwargs):
important method in all of BBOT. It is basically the central intersection that
every event passes through.
- Probably it is also needless to say that it exists in a delicate balance.
- Close to half of my debugging time has been spent in this function.
- I have slain many dragons here and there may still be more yet to slay.
+ It exists in a delicate balance. Close to half of my debugging time has been spent
+ in this function. I have slain many dragons here and there may still be more yet to slay.
Tread carefully, friend. -TheTechromancer
@@ -150,7 +170,6 @@ async def _emit_event(self, event, **kwargs):
- Updating scan statistics.
"""
log.debug(f"Emitting {event}")
- distribute_event = True
event_distributed = False
try:
on_success_callback = kwargs.pop("on_success_callback", None)
@@ -187,11 +206,6 @@ async def _emit_event(self, event, **kwargs):
)
dns_children = {}
- # We do this again in case event.data changed during resolve_event()
- if event.type == "DNS_NAME" and not self._event_precheck(event, exclude=()):
- log.debug(f"Omitting due to failed precheck: {event}")
- distribute_event = False
-
if event.type in ("DNS_NAME", "IP_ADDRESS"):
for tag in dns_tags:
event.add_tag(tag)
@@ -202,14 +216,11 @@ async def _emit_event(self, event, **kwargs):
event_blacklisted = event_blacklisted_dns | self.scan.blacklisted(event)
if event_blacklisted:
event.add_tag("blacklisted")
-
- # Blacklist purging
- if "blacklisted" in event.tags:
reason = "event host"
if event_blacklisted_dns:
reason = "DNS associations"
log.debug(f"Omitting due to blacklisted {reason}: {event}")
- distribute_event = False
+ return
# DNS_NAME --> DNS_NAME_UNRESOLVED
if event.type == "DNS_NAME" and "unresolved" in event.tags and not "target" in event.tags:
@@ -219,31 +230,18 @@ async def _emit_event(self, event, **kwargs):
await self.scan.helpers.cloud.tag_event(event)
# Scope shepherding
- # here, we buff or nerf the scope distance of an event based on its attributes and certain scan settings
- event_is_duplicate = self.is_duplicate_event(event)
+ # here is where we make sure in-scope events are set to their proper scope distance
+ if event.host and event_whitelisted:
+ log.debug(f"Making {event} in-scope")
+ event.scope_distance = 0
+
event_in_report_distance = event.scope_distance <= self.scan.scope_report_distance
- set_scope_distance = event.scope_distance
- if event_whitelisted:
- set_scope_distance = 0
- if event.host:
- # here, we evaluate some weird logic
- # the reason this exists is to ensure we don't have orphans in the graph
- # because forcefully internalizing certain events can orphan their children
- event_will_be_output = event_whitelisted or event_in_report_distance
- event_is_duplicate = event_is_duplicate and not event._force_output
- if event_will_be_output and not event_is_duplicate:
- if set_scope_distance == 0:
- log.debug(f"Making {event} in-scope")
- source_trail = event.set_scope_distance(set_scope_distance)
- # force re-emit internal source events
- for s in source_trail:
- self.queue_event(s)
- else:
- if event.scope_distance > self.scan.scope_report_distance:
- log.debug(
- f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})"
- )
- event.make_internal()
+ event_will_be_output = event.always_emit or event_in_report_distance
+ if not event_will_be_output:
+ log.debug(
+ f"Making {event} internal because its scope_distance ({event.scope_distance}) > scope_report_distance ({self.scan.scope_report_distance})"
+ )
+ event.internal = True
# check for wildcards
if event.scope_distance <= self.scan.scope_search_distance:
@@ -251,6 +249,24 @@ async def _emit_event(self, event, **kwargs):
if not self.scan.helpers.is_ip_type(event.host):
await self.scan.helpers.dns.handle_wildcard_event(event, dns_children)
+ # For DNS_NAMEs, we've waited to do this until now, in case event.data changed during handle_wildcard_event()
+ if event.type == "DNS_NAME":
+ acceptable = self._event_precheck(event)
+ if not acceptable:
+ return
+
+ # if we discovered something interesting from an internal event,
+ # make sure we preserve its chain of parents
+ source = event.source
+ if source.internal and (event_will_be_output or event._graph_important):
+ source_in_report_distance = source.scope_distance <= self.scan.scope_report_distance
+ if source_in_report_distance:
+ source.internal = False
+ if not source._graph_important:
+ source._graph_important = True
+ log.debug(f"Re-queuing internal event {source} with parent {event}")
+ self.queue_event(source)
+
# now that the event is properly tagged, we can finally make decisions about it
abort_result = False
if callable(abort_if):
@@ -264,26 +280,13 @@ async def _emit_event(self, event, **kwargs):
log.debug(msg)
return
- if not self.accept_event(event):
- return
-
# run success callback before distributing event (so it can add tags, etc.)
- if distribute_event:
- if callable(on_success_callback):
- async with self.scan._acatch(context=on_success_callback):
- await self.scan.helpers.execute_sync_or_async(on_success_callback, event)
-
- if not event.host or (event.always_emit and not event_is_duplicate):
- log.debug(
- f"Force-emitting {event} (host:{event.host}, always_emit={event.always_emit}, is_duplicate={event_is_duplicate})"
- )
- source_trail = event.unmake_internal(force_output=True)
- for s in source_trail:
- self.queue_event(s)
+ if callable(on_success_callback):
+ async with self.scan._acatch(context=on_success_callback):
+ await self.scan.helpers.execute_sync_or_async(on_success_callback, event)
- if distribute_event:
- await self.distribute_event(event)
- event_distributed = True
+ await self.distribute_event(event)
+ event_distributed = True
# speculate DNS_NAMES and IP_ADDRESSes from other event types
source_event = event
@@ -304,13 +307,13 @@ async def _emit_event(self, event, **kwargs):
### Emit DNS children ###
if self.dns_resolution:
- emit_children = -1 < event.scope_distance < self.scan.scope_dns_search_distance
- if emit_children:
- # only emit DNS children once for each unique host
- host_hash = hash(str(event.host))
- if host_hash in self.events_accepted:
- emit_children = False
- self.events_accepted.add(host_hash)
+ emit_children = True
+ in_dns_scope = -1 < event.scope_distance < self.scan.scope_dns_search_distance
+ # only emit DNS children once for each unique host
+ host_hash = hash(str(event.host))
+ if host_hash in self.outgoing_dup_tracker:
+ emit_children = False
+ self.outgoing_dup_tracker.add(host_hash)
if emit_children:
dns_child_events = []
@@ -323,7 +326,9 @@ async def _emit_event(self, event, **kwargs):
child_event = self.scan.make_event(
record, "DNS_NAME", module=module, source=source_event
)
- dns_child_events.append(child_event)
+ host_hash = hash(str(child_event.host))
+ if in_dns_scope or self.scan.in_scope(child_event):
+ dns_child_events.append(child_event)
except ValidationError as e:
log.warning(
f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}'
@@ -341,9 +346,9 @@ async def _emit_event(self, event, **kwargs):
self.scan.stats.event_distributed(event)
log.debug(f"{event.module}.emit_event() finished for {event}")
- def hash_event(self, event):
+ def hash_event_graph(self, event):
"""
- Hash an event for duplicate detection
+ Hash an event for graph duplicate detection
This is necessary because duplicate events from certain sources (e.g. DNS)
need to be allowed in order to preserve their relationship trail
@@ -355,23 +360,35 @@ def hash_event(self, event):
else:
return hash((event, str(event.module)))
- def is_duplicate_event(self, event, add=False):
+ def is_incoming_duplicate(self, event, add=False):
"""
- Calculate whether an event is a duplicate on a per-module basis
+ Calculate whether an event is a duplicate in the context of the module that emitted it
+ This will return True if the event's parent module has raised the event before.
"""
- event_hash = self.hash_event(event)
- suppress_dupes = getattr(event.module, "suppress_dupes", True)
- duplicate_event = suppress_dupes and event_hash in self.events_accepted
+ try:
+ event_hash = event.module._outgoing_dedup_hash(event)
+ except AttributeError:
+ event_hash = hash(event)
+ is_dup = event_hash in self.incoming_dup_tracker
if add:
- self.events_accepted.add(event_hash)
- return duplicate_event
+ self.incoming_dup_tracker.add(event_hash)
+ suppress_dupes = getattr(event.module, "suppress_dupes", True)
+ if suppress_dupes and is_dup:
+ return True
+ return False
- def accept_event(self, event):
- is_duplicate = self.is_duplicate_event(event, add=True)
- if is_duplicate and not event._force_output:
- log.debug(f"{event.module}: not raising duplicate event {event}")
- return False
- return True
+ def is_outgoing_duplicate(self, event, add=False):
+ """
+ Calculate whether an event is a duplicate in the context of the whole scan,
+ This will return True if the same event (irregardless of its source module) has been emitted before.
+
+ TODO: Allow modules to use this for custom deduplication such as on a per-host or per-domain basis.
+ """
+ event_hash = hash(event)
+ is_dup = event_hash in self.outgoing_dup_tracker
+ if add:
+ self.outgoing_dup_tracker.add(event_hash)
+ return is_dup
async def distribute_event(self, *args, **kwargs):
"""
@@ -379,18 +396,17 @@ async def distribute_event(self, *args, **kwargs):
"""
async with self.scan._acatch(context=self.distribute_event):
event = self.scan.make_event(*args, **kwargs)
-
- event_hash = hash(event)
- dup = event_hash in self.events_distributed
- if dup:
+ is_outgoing_duplicate = self.is_outgoing_duplicate(event)
+ if is_outgoing_duplicate:
self.scan.verbose(f"{event.module}: Duplicate event: {event}")
- else:
- self.events_distributed.add(event_hash)
# absorb event into the word cloud if it's in scope
- if not dup and -1 < event.scope_distance < 1:
+ if not is_outgoing_duplicate and -1 < event.scope_distance < 1:
self.scan.word_cloud.absorb_event(event)
for mod in self.scan.modules.values():
- if not dup or mod.accept_dupes or (mod._type == "output" and event._force_output):
+ acceptable_dup = (not is_outgoing_duplicate) or mod.accept_dupes
+ # graph_important = mod._type == "output" and event._graph_important == True
+ graph_important = mod._preserve_graph and event._graph_important
+ if acceptable_dup or graph_important:
await mod.queue_event(event)
async def _worker_loop(self):
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index ecc4d31e4..84c623300 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -771,12 +771,6 @@ def make_event(self, *args, **kwargs):
event = make_event(*args, **kwargs)
return event
- @property
- def log(self):
- if self._log is None:
- self._log = logging.getLogger(f"bbot.agent.scanner")
- return self._log
-
@property
def root_event(self):
"""
@@ -1029,19 +1023,6 @@ async def _status_ticker(self, interval=15):
await asyncio.sleep(interval)
self.manager.modules_status(_log=True)
- @contextlib.contextmanager
- def _catch(self, context="scan", finally_callback=None):
- """
- Handle common errors by stopping scan, logging tracebacks, etc.
-
- with catch():
- do_stuff()
- """
- try:
- yield
- except BaseException as e:
- self._handle_exception(e, context=context)
-
@contextlib.asynccontextmanager
async def _acatch(self, context="scan", finally_callback=None):
"""
diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py
index dc98ca95d..20e8bbf64 100644
--- a/bbot/scanner/target.py
+++ b/bbot/scanner/target.py
@@ -128,7 +128,7 @@ def add_target(self, t):
t, source=self.scan.root_event, module=self._dummy_module, tags=["target"]
)
if self.make_in_scope:
- event.set_scope_distance(0)
+ event.scope_distance = 0
try:
self._events[event.host].add(event)
except KeyError:
diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py
index 2c74fe190..a08ef3ece 100644
--- a/bbot/test/bbot_fixtures.py
+++ b/bbot/test/bbot_fixtures.py
@@ -210,7 +210,7 @@ class bbot_events:
]
for e in bbot_events.all:
- e.set_scope_distance(0)
+ e.scope_distance = 0
return bbot_events
diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py
index 5976786ef..48d1d43d0 100644
--- a/bbot/test/test_step_1/test_events.py
+++ b/bbot/test/test_step_1/test_events.py
@@ -142,7 +142,7 @@ async def test_events(events, scan, helpers, bbot_config):
# scope distance
event1 = scan.make_event("1.2.3.4", dummy=True)
assert event1._scope_distance == -1
- event1.set_scope_distance(0)
+ event1.scope_distance = 0
assert event1._scope_distance == 0
event2 = scan.make_event("2.3.4.5", source=event1)
assert event2._scope_distance == 1
@@ -157,20 +157,7 @@ async def test_events(events, scan, helpers, bbot_config):
root_event = scan.make_event("0.0.0.0", dummy=True)
internal_event1 = scan.make_event("1.2.3.4", source=root_event, internal=True)
assert internal_event1._internal == True
- assert internal_event1._made_internal == True
- internal_event1.set_scope_distance(0)
- assert internal_event1._internal == False
- assert internal_event1._made_internal == False
- internal_event2 = scan.make_event("2.3.4.5", source=internal_event1, internal=True)
- internal_event3 = scan.make_event("3.4.5.6", source=internal_event2, internal=True)
- internal_event4 = scan.make_event("4.5.6.7", source=internal_event3)
- source_trail = internal_event4.set_scope_distance(0)
- assert internal_event4._internal == False
- assert internal_event3._internal == False
- assert internal_event2._internal == False
- assert len(source_trail) == 2
- assert internal_event2 in source_trail
- assert internal_event3 in source_trail
+ assert "internal" in internal_event1.tags
# event sorting
parent1 = scan.make_event("127.0.0.1", source=scan.root_event)
diff --git a/bbot/test/test_step_1/test_manager.py b/bbot/test/test_step_1/test_manager.py
deleted file mode 100644
index 16e6db7f5..000000000
--- a/bbot/test/test_step_1/test_manager.py
+++ /dev/null
@@ -1,210 +0,0 @@
-from ..bbot_fixtures import * # noqa: F401
-
-
-@pytest.mark.asyncio
-async def test_manager(bbot_config, bbot_scanner):
- dns_config = OmegaConf.merge(
- default_config, OmegaConf.create({"dns_resolution": True, "scope_report_distance": 1})
- )
-
- # test _emit_event
- results = []
- output = []
- event_children = []
-
- async def results_append(e):
- results.append(e)
-
- async def output_append(e):
- output.append(e)
-
- def event_children_append(e):
- event_children.append(e)
-
- success_callback = lambda e: results.append("success")
- scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor"], output_modules=["json"], config=dns_config)
- await scan1.load_modules()
- module = scan1.modules["ipneighbor"]
- module.scope_distance_modifier = 0
- module.queue_event = results_append
- output_module = scan1.modules["json"]
- output_module.queue_event = output_append
- scan1.status = "RUNNING"
- manager = scan1.manager
- # manager.distribute_event = lambda e: results.append(e)
- localhost = scan1.make_event("127.0.0.1", source=scan1.root_event, tags=["localhost"])
-
- class DummyModule1:
- _type = "output"
- suppress_dupes = True
-
- class DummyModule2:
- _type = "DNS"
- suppress_dupes = True
-
- class DummyModule3:
- _type = "DNS"
- suppress_dupes = True
-
- localhost.module = DummyModule1()
- # make sure abort_if works as intended
- await manager._emit_event(localhost, abort_if=lambda e: e.module._type == "output")
- assert len(results) == 0
- manager.events_accepted.clear()
- manager.events_distributed.clear()
- await manager._emit_event(localhost, abort_if=lambda e: e.module._type != "output")
- assert len(results) == 1
- results.clear()
- manager.events_accepted.clear()
- manager.events_distributed.clear()
- # make sure success_callback works as intended
- await manager._emit_event(
- localhost, on_success_callback=success_callback, abort_if=lambda e: e.module._type == "plumbus"
- )
- assert localhost in results
- assert "success" in results
- results.clear()
- # make sure deduplication is working
- localhost2 = scan1.make_event("127.0.0.1", source=scan1.root_event, tags=["localhost2"])
- manager._emit_event(localhost2)
- assert len(results) == 0
- # make sure dns resolution is working
- googledns = scan1.make_event("8.8.8.8", source=scan1.root_event)
- googledns.module = DummyModule2()
- googledns.source = "asdf"
- googledns.set_scope_distance(0)
- manager.queue_event = event_children_append
- await manager._emit_event(googledns)
- assert len(event_children) > 0
- assert googledns in results
- assert googledns in output
- results.clear()
- output.clear()
- event_children.clear()
- # make sure deduplication catches the same event
- await manager._emit_event(googledns)
- assert len(output) == 0
- assert len(results) == 0
- assert len(event_children) == 0
- output.clear()
- event_children.clear()
- # make sure _force_output overrides dup detection
- googledns._force_output = True
- await manager._emit_event(googledns)
- assert googledns in output
- assert len(event_children) == 0
- googledns._force_output = False
- results.clear()
- event_children.clear()
- # same dns event but different source
- source_event = manager.scan.make_event("1.2.3.4", "IP_ADDRESS", source=manager.scan.root_event)
- source_event._resolved.set()
- googledns.source = source_event
- await manager._emit_event(googledns)
- assert len(event_children) == 0
- assert googledns in output
-
- # error catching
- msg = "Ignore this error, it belongs here"
- exceptions = (Exception(msg), KeyboardInterrupt(msg), BrokenPipeError(msg))
- for e in exceptions:
- with manager.scan._catch():
- raise e
-
-
-@pytest.mark.asyncio
-async def test_scope_distance(bbot_scanner, bbot_config):
- # event filtering based on scope_distance
- scan1 = bbot_scanner(
- "127.0.0.1", "evilcorp.com", modules=["ipneighbor"], output_modules=["json"], config=bbot_config
- )
- scan1.status = "RUNNING"
- await scan1.load_modules()
- module = scan1.modules["ipneighbor"]
- module_queue = module.incoming_event_queue._queue
- output_module = scan1.modules["json"]
- output_queue = output_module.incoming_event_queue._queue
- manager = scan1.manager
- test_event1 = scan1.make_event("127.0.0.1", source=scan1.root_event)
-
- assert scan1.scope_search_distance == 0
- assert scan1.scope_report_distance == 0
- assert module.scope_distance_modifier == 1
-
- # test _emit_event() with scope_distance == 0
- await manager._emit_event(test_event1)
- assert test_event1.scope_distance == 0
- assert test_event1._internal == False
- assert test_event1 in output_queue
- assert test_event1 in module_queue
-
- test_event2 = scan1.make_event("2.3.4.5", source=test_event1)
- test_event3 = scan1.make_event("3.4.5.6", source=test_event2)
- test_event4 = scan1.make_event("4.5.6.7", source=test_event2)
- test_event4._force_output = True
- dns_event = scan1.make_event("evilcorp.com", source=scan1.root_event)
-
- # non-watched event type
- await manager._emit_event(dns_event)
- assert dns_event.scope_distance == 0
- assert dns_event in output_queue
- assert dns_event not in module_queue
-
- # test _emit_event() with scope_distance == 1
- assert test_event2.scope_distance == 1
- await manager._emit_event(test_event2)
- assert test_event2.scope_distance == 1
- assert test_event2._internal == True
- assert test_event2 not in output_queue
- assert test_event2 in module_queue
- valid, reason = await module._event_postcheck(test_event2)
- assert valid
-
- # test _emit_event() with scope_distance == 2
- assert test_event3.scope_distance == 2
- await manager._emit_event(test_event3)
- assert test_event3.scope_distance == 2
- assert test_event3._internal == True
- assert test_event3 not in output_queue
- assert test_event3 in module_queue
- valid, reason = await module._event_postcheck(test_event3)
- assert not valid
- assert reason.startswith("its scope_distance (2) exceeds the maximum allowed by the scan")
-
- # test _emit_event() with scope_distance == 2 and _force_output == True
- assert test_event4.scope_distance == 2
- await manager._emit_event(test_event4)
- assert test_event4.scope_distance == 2
- assert test_event4._internal == True
- assert test_event4._force_output == True
- assert test_event4 in output_queue
- assert test_event4 in module_queue
- valid, reason = await module._event_postcheck(test_event4)
- assert not valid
- assert reason.startswith("its scope_distance (2) exceeds the maximum allowed by the scan")
-
- # test _always_emit == True
- geoevent = scan1.make_event("USA", "GEOLOCATION", source=test_event3)
- assert geoevent.scope_distance == 3
- assert geoevent.always_emit == True
- assert geoevent._force_output == False
- await manager._emit_event(geoevent)
- assert geoevent._force_output == True
- assert geoevent in output_queue
- assert geoevent not in module_queue
-
- # test always_emit tag
- affiliate_event = scan1.make_event("5.6.7.8", source=test_event3, tags="affiliate")
- assert "affiliate" in affiliate_event.tags
- assert "affiliate" in affiliate_event._always_emit_tags
- assert affiliate_event.scope_distance == 3
- assert affiliate_event._always_emit == False
- assert affiliate_event.always_emit == True
- assert affiliate_event._force_output == False
- await manager._emit_event(affiliate_event)
- assert affiliate_event._force_output == True
- assert affiliate_event in output_queue
- assert affiliate_event in module_queue
- valid, reason = await module._event_postcheck(affiliate_event)
- assert not valid
- assert reason.startswith("its scope_distance (3) exceeds the maximum allowed by the scan")
diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py
new file mode 100644
index 000000000..490ded331
--- /dev/null
+++ b/bbot/test/test_step_1/test_manager_deduplication.py
@@ -0,0 +1,178 @@
+from ..bbot_fixtures import * # noqa: F401
+from bbot.modules.base import BaseModule
+
+
+@pytest.mark.asyncio
+async def test_manager_deduplication(bbot_config, bbot_scanner):
+
+ class DefaultModule(BaseModule):
+ _name = "default_module"
+ watched_events = ["DNS_NAME"]
+
+ async def setup(self):
+ self.events = []
+ return True
+
+ async def handle_event(self, event):
+ self.events.append(event)
+ self.emit_event(f"{self.name}.test.notreal", "DNS_NAME", source=event)
+
+ class EverythingModule(DefaultModule):
+ _name = "everything_module"
+ watched_events = ["*"]
+ scope_distance_modifier = 10
+ accept_dupes = True
+ suppress_dupes = False
+
+ async def handle_event(self, event):
+ self.events.append(event)
+ if event.type == "DNS_NAME":
+ self.emit_event(f"{event.data}:88", "OPEN_TCP_PORT", source=event)
+
+ class NoSuppressDupes(DefaultModule):
+ _name = "no_suppress_dupes"
+ suppress_dupes = False
+
+ class AcceptDupes(DefaultModule):
+ _name = "accept_dupes"
+ accept_dupes = True
+
+ class PerHostOnly(DefaultModule):
+ _name = "per_host_only"
+ per_host_only = True
+
+ class PerDomainOnly(DefaultModule):
+ _name = "per_domain_only"
+ per_domain_only = True
+
+
+ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs):
+ merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(_config))
+ scan = bbot_scanner(*args, config=merged_config, **kwargs)
+ default_module = DefaultModule(scan)
+ everything_module = EverythingModule(scan)
+ no_suppress_dupes = NoSuppressDupes(scan)
+ accept_dupes = AcceptDupes(scan)
+ per_host_only = PerHostOnly(scan)
+ per_domain_only = PerDomainOnly(scan)
+ scan.modules["default_module"] = default_module
+ scan.modules["everything_module"] = everything_module
+ scan.modules["no_suppress_dupes"] = no_suppress_dupes
+ scan.modules["accept_dupes"] = accept_dupes
+ scan.modules["per_host_only"] = per_host_only
+ scan.modules["per_domain_only"] = per_domain_only
+ if _dns_mock:
+ scan.helpers.dns.mock_dns(_dns_mock)
+ if scan_callback is not None:
+ scan_callback(scan)
+ return (
+ [e async for e in scan.async_start()],
+ default_module.events,
+ everything_module.events,
+ no_suppress_dupes.events,
+ accept_dupes.events,
+ per_host_only.events,
+ per_domain_only.events,
+ )
+
+ dns_mock_chain = {
+ ("default_module.test.notreal", "A"): "127.0.0.3",
+ ("everything_module.test.notreal", "A"): "127.0.0.4",
+ ("no_suppress_dupes.test.notreal", "A"): "127.0.0.5",
+ ("accept_dupes.test.notreal", "A"): "127.0.0.6",
+ ("per_host_only.test.notreal", "A"): "127.0.0.7",
+ ("per_domain_only.test.notreal", "A"): "127.0.0.8",
+ }
+
+ # dns search distance = 1, report distance = 0
+ events, default_events, all_events, no_suppress_dupes, accept_dupes, per_host_only, per_domain_only = await do_scan(
+ "test.notreal",
+ _config={"dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0},
+ _dns_mock=dns_mock_chain,
+ )
+
+ for _ in "events", "default_events", "all_events", "no_suppress_dupes", "accept_dupes", "per_host_only", "per_domain_only":
+ _events = locals()[_]
+ log.critical(f"========== {_} ==========")
+ for e in _events:
+ log.critical(e)
+
+ assert len(events) == 21
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_host_only.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "per_host_only.test.notreal" and str(e.module) == "per_host_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_domain_only.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "per_host_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_host_only.test.notreal"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "test.notreal"])
+ assert 5 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "no_suppress_dupes.test.notreal"])
+
+ assert len(default_events) == 6
+ assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_host_only.test.notreal" and str(e.module) == "per_host_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+
+ assert len(all_events) == 26
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_host_only.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_host_only.test.notreal" and str(e.module) == "per_host_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.source.data == "default_module.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.source.data == "no_suppress_dupes.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.source.data == "accept_dupes.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.source.data == "per_host_only.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.8" and str(e.module) == "A" and e.source.data == "per_domain_only.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "accept_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "accept_dupes.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "default_module.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "default_module.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_domain_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_domain_only.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "per_host_only.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "per_host_only.test.notreal"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "test.notreal"])
+ assert 5 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "no_suppress_dupes.test.notreal:88" and str(e.module) == "everything_module" and e.source.data == "no_suppress_dupes.test.notreal"])
+
+ assert len(no_suppress_dupes) == 6
+ assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "per_host_only.test.notreal" and str(e.module) == "per_host_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in no_suppress_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+
+ assert len(accept_dupes) == 10
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "accept_dupes.test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "default_module.test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_domain_only.test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "per_host_only.test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "per_host_only.test.notreal" and str(e.module) == "per_host_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in accept_dupes if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+
+ assert len(per_host_only) == 6
+ assert 1 == len([e for e in per_host_only if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in per_host_only if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in per_host_only if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in per_host_only if e.type == "DNS_NAME" and e.data == "per_domain_only.test.notreal" and str(e.module) == "per_domain_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in per_host_only if e.type == "DNS_NAME" and e.data == "per_host_only.test.notreal" and str(e.module) == "per_host_only" and e.source.data == "test.notreal"])
+ assert 1 == len([e for e in per_host_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
+
+ assert len(per_domain_only) == 1
+ assert 1 == len([e for e in per_domain_only if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "TARGET" and "SCAN:" in e.source.data])
diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py
new file mode 100644
index 000000000..d3883393f
--- /dev/null
+++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py
@@ -0,0 +1,828 @@
+from ..bbot_fixtures import * # noqa: F401
+
+from pytest_httpserver import HTTPServer
+
+
+@pytest.fixture
+def bbot_other_httpservers():
+
+ server_hosts = [
+ ("127.0.0.77", 8888),
+ ("127.0.0.88", 8888),
+ ("127.0.0.99", 8888),
+ ("127.0.0.111", 8888),
+ ("127.0.0.222", 8889),
+ ("127.0.0.33", 8889),
+ ]
+
+ servers = [HTTPServer(host=host, port=port) for host, port in server_hosts]
+ for server in servers:
+ server.start()
+
+ yield servers
+
+ for server in servers:
+ server.clear()
+ if server.is_running():
+ server.stop()
+ server.check_assertions()
+ server.clear()
+
+
+
+@pytest.mark.asyncio
+async def test_manager_scope_accuracy(bbot_config, bbot_scanner, bbot_httpserver, bbot_other_httpservers, bbot_httpserver_ssl):
+ """
+ This test ensures that BBOT correctly handles different scope distance settings.
+ It performs these tests for normal modules, output modules, and their graph variants,
+ ensuring that when an internal event leads to an interesting discovery, the entire event chain is preserved.
+ This is important for preventing orphans in the graph.
+ """
+
+ from bbot.modules.base import BaseModule
+ from bbot.modules.output.base import BaseOutputModule
+
+ server_77, server_88, server_99, server_111, server_222, server_33 = bbot_other_httpservers
+
+ bbot_httpserver.expect_request(uri="/").respond_with_data(response_data="")
+ server_77.expect_request(uri="/").respond_with_data(response_data="")
+ server_88.expect_request(uri="/").respond_with_data(response_data="")
+ server_99.expect_request(uri="/").respond_with_data(response_data="")
+ server_111.expect_request(uri="/").respond_with_data(response_data="")
+ server_222.expect_request(uri="/").respond_with_data(response_data="")
+ server_33.expect_request(uri="/").respond_with_data(response_data="")
+
+ class DummyModule(BaseModule):
+ _name = "dummy_module"
+ watched_events = ["*"]
+ scope_distance_modifier = 10
+ accept_dupes = True
+
+ async def setup(self):
+ self.events = []
+ return True
+
+ async def handle_event(self, event):
+ self.events.append(event)
+
+ class DummyModuleNoDupes(DummyModule):
+ accept_dupes = False
+
+ class DummyGraphOutputModule(BaseOutputModule):
+ _name = "dummy_graph_output_module"
+ watched_events = ["*"]
+ _preserve_graph = True
+
+ async def setup(self):
+ self.events = []
+ return True
+
+ async def handle_event(self, event):
+ self.events.append(event)
+
+ class DummyGraphBatchOutputModule(DummyGraphOutputModule):
+ _name = "dummy_graph_batch_output_module"
+ watched_events = ["*"]
+ _preserve_graph = True
+ batch_size = 5
+
+ async def handle_batch(self, *events):
+ for event in events:
+ self.events.append(event)
+
+ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs):
+ merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(_config))
+ scan = bbot_scanner(*args, config=merged_config, **kwargs)
+ dummy_module = DummyModule(scan)
+ dummy_module_nodupes = DummyModuleNoDupes(scan)
+ dummy_graph_output_module = DummyGraphOutputModule(scan)
+ dummy_graph_batch_output_module = DummyGraphBatchOutputModule(scan)
+ scan.modules["dummy_module"] = dummy_module
+ scan.modules["dummy_module_nodupes"] = dummy_module_nodupes
+ scan.modules["dummy_graph_output_module"] = dummy_graph_output_module
+ scan.modules["dummy_graph_batch_output_module"] = dummy_graph_batch_output_module
+ if _dns_mock:
+ scan.helpers.dns.mock_dns(_dns_mock)
+ if scan_callback is not None:
+ scan_callback(scan)
+ return (
+ [e async for e in scan.async_start()],
+ dummy_module.events,
+ dummy_module_nodupes.events,
+ dummy_graph_output_module.events,
+ dummy_graph_batch_output_module.events,
+ )
+
+ dns_mock_chain = {
+ ("test.notreal", "A"): "127.0.0.66",
+ ("127.0.0.66", "PTR"): "test.notrealzies",
+ ("test.notrealzies", "CNAME"): "www.test.notreal",
+ ("www.test.notreal", "A"): "127.0.0.77",
+ ("127.0.0.77", "PTR"): "test2.notrealzies",
+ ("test2.notrealzies", "A"): "127.0.0.88",
+ }
+
+ # dns search distance = 1, report distance = 0
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "test.notreal",
+ _config={"dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0},
+ _dns_mock=dns_mock_chain,
+ )
+
+ assert len(events) == 2
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.test.notreal"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"])
+
+ for _all_events in (all_events, all_events_nodups):
+ assert len(_all_events) == 3
+ assert 1 == len([e for e in _all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in _all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"])
+ assert 0 == len([e for e in _all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"])
+ assert 0 == len([e for e in _all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"])
+
+ assert len(graph_output_events) == 2
+ assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"])
+ assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies"])
+ assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.test.notreal"])
+ assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"])
+
+ # dns search distance = 2, report distance = 0
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "test.notreal",
+ _config={"dns_resolution": True, "scope_dns_search_distance": 2, "scope_report_distance": 0},
+ _dns_mock=dns_mock_chain,
+ )
+
+ assert len(events) == 3
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ assert len(all_events) == 9
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ assert len(all_events_nodups) == 7
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 5
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ # dns search distance = 2, report distance = 1
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "test.notreal",
+ _config={"dns_resolution": True, "scope_dns_search_distance": 2, "scope_report_distance": 1},
+ _dns_mock=dns_mock_chain,
+ )
+
+ assert len(events) == 6
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ assert len(all_events) == 8
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ assert len(all_events_nodups) == 7
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test2.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 0 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 6
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "www.test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test2.notrealzies"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+
+ dns_mock_chain = {
+ ("test.notreal", "A"): "127.0.0.66",
+ ("127.0.0.66", "PTR"): "test.notrealzies",
+ ("test.notrealzies", "A"): "127.0.0.77",
+ }
+
+ class DummyVulnModule(BaseModule):
+ _name = "dummyvulnmodule"
+ watched_events = ["IP_ADDRESS"]
+ scope_distance_modifier = 3
+ accept_dupes = True
+
+ async def filter_event(self, event):
+ if event.data == "127.0.0.77":
+ return True
+ return False, "bleh"
+
+ async def handle_event(self, event):
+ self.emit_event(
+ {"host": str(event.host), "description": "yep", "severity": "CRITICAL"}, "VULNERABILITY", source=event
+ )
+
+ def custom_setup(scan):
+ dummyvulnmodule = DummyVulnModule(scan)
+ scan.modules["dummyvulnmodule"] = dummyvulnmodule
+
+ # dns search distance = 3, report distance = 1
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "test.notreal",
+ scan_callback=custom_setup,
+ _config={"dns_resolution": True, "scope_dns_search_distance": 3, "scope_report_distance": 1},
+ _dns_mock=dns_mock_chain,
+ )
+
+ assert len(events) == 4
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notrealzies"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77"])
+ assert 1 == len([e for e in events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e.internal == False and e.scope_distance == 3])
+
+ assert len(all_events) == 8
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == True and e.scope_distance == 3])
+ assert 1 == len([e for e in all_events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e.internal == False and e.scope_distance == 3])
+
+ assert len(all_events_nodups) == 6
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == True and e.scope_distance == 3])
+ assert 1 == len([e for e in all_events_nodups if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e.internal == False and e.scope_distance == 3])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 6
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.66" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notrealzies" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == True and e.scope_distance == 3])
+ assert 1 == len([e for e in _graph_output_events if e.type == "VULNERABILITY" and e.data["host"] == "127.0.0.77" and e.internal == False and e.scope_distance == 3])
+
+ # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 0
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "127.0.0.1/31",
+ modules=["httpx", "excavate"],
+ _config={
+ "scope_search_distance": 0,
+ "scope_dns_search_distance": 2,
+ "scope_report_distance": 1,
+ "speculate": True,
+ "internal_modules": {"speculate": {"ports": "8888"}},
+ "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"],
+ },
+ )
+
+ assert len(events) == 6
+ assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"])
+
+ assert len(all_events) == 13
+ assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == True and e.scope_distance == 1])
+
+ assert len(all_events_nodups) == 11
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == True and e.scope_distance == 1])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 6
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888"])
+
+ # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 0, in_scope_only = False
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "127.0.0.1/31",
+ modules=["httpx", "excavate"],
+ _config={
+ "scope_search_distance": 0,
+ "scope_dns_search_distance": 2,
+ "scope_report_distance": 1,
+ "speculate": True,
+ "modules": {"httpx": {"in_scope_only": False}},
+ "internal_modules": {"speculate": {"ports": "8888"}},
+ "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"],
+ },
+ )
+
+ assert len(events) == 8
+ assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+
+ assert len(all_events) == 18
+ assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
+
+ assert len(all_events_nodups) == 15
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 8
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/"])
+
+ # httpx/speculate IP_RANGE --> IP_ADDRESS --> OPEN_TCP_PORT --> URL, search distance = 1
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "127.0.0.1/31",
+ modules=["httpx", "excavate"],
+ _config={
+ "scope_search_distance": 1,
+ "scope_dns_search_distance": 2,
+ "scope_report_distance": 1,
+ "speculate": True,
+ "modules": {"httpx": {"in_scope_only": False}},
+ "internal_modules": {"speculate": {"ports": "8888"}},
+ "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"],
+ },
+ )
+
+ assert len(events) == 8
+ assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+
+ assert len(all_events) == 23
+ assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.88:8888" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.88:8888" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.99" and e.internal == True and e.scope_distance == 3])
+
+ assert len(all_events_nodups) == 20
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.88" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.88:8888" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.88:8888/" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.88:8888" and e.internal == True and e.scope_distance == 2])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.99:8888/" and e.internal == True and e.scope_distance == 3])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.99" and e.internal == True and e.scope_distance == 3])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 8
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:8888"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.1:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.1:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.77:8888/"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.77" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.77:8888" and e.internal == False and e.scope_distance == 1])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.77:8888/" and e.internal == False and e.scope_distance == 1])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.77:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.88"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.88:8888/"])
+
+ # 2 events from a single HTTP_RESPONSE
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "127.0.0.111/31",
+ whitelist=["127.0.0.111/31", "127.0.0.222", "127.0.0.33"],
+ modules=["httpx", "excavate"],
+ output_modules=["python"],
+ _config={
+ "scope_search_distance": 0,
+ "scope_dns_search_distance": 2,
+ "scope_report_distance": 0,
+ "speculate": True,
+ "internal_modules": {"speculate": {"ports": "8888"}},
+ "omit_event_types": ["HTTP_RESPONSE", "URL_UNVERIFIED"],
+ },
+ )
+
+ assert len(events) == 11
+ assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.110:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.111:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.222" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.44:8888/" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.44" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.55:8888/" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.55" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1])
+
+ assert len(all_events) == 30
+ assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.110:8888" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.111:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.222" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.44:8888/" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.44" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.55:8888/" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.55" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1])
+
+ assert len(all_events_nodups) == 26
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.110" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.110:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.111:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.222" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.33" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.44:8888/" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.44" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.55:8888/" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.55" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888" and e.internal == True and e.scope_distance == 1])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 11
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.110/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.110"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.111" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.110:8888"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.111:8888" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.111:8888/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.111:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.222:8889/"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.222"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.33:8889/"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.33"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8888"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.222:8889" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8888"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.33:8889" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.222:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.222:8889"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "URL" and e.data == "http://127.0.0.33:8889/" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "HTTP_RESPONSE" and e.data["input"] == "127.0.0.33:8889"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.44:8888/"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.44"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "URL_UNVERIFIED" and e.data == "http://127.0.0.55:8888/"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.55"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.44:8888"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.55:8888"])
+
+ # sslcert with in-scope chain
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "127.0.0.0/31",
+ modules=["speculate", "sslcert"],
+ _config={"dns_resolution": False, "scope_report_distance": 0, "internal_modules": {"speculate": {"ports": "9999"}}},
+ _dns_mock={("www.bbottest.notreal", "A"): "127.0.1.0", ("test.notreal", "A"): "127.0.0.1"},
+ )
+
+ assert len(events) == 5
+ assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
+ assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
+
+ assert len(all_events) == 14
+ assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 0])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 1 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "www.bbottest.notreal:9999" and e.internal == True and e.scope_distance == 1 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "bbottest.notreal" and e.internal == True and e.scope_distance == 2 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal" and e.internal == True and e.scope_distance == 1 and str(e.module) == "speculate"])
+
+ assert len(all_events_nodups) == 12
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 1 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "www.bbottest.notreal:9999" and e.internal == True and e.scope_distance == 1 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME_UNRESOLVED" and e.data == "bbottest.notreal" and e.internal == True and e.scope_distance == 2 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal" and e.internal == True and e.scope_distance == 1 and str(e.module) == "speculate"])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 5
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
+ assert 1 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "www.bbottest.notreal:9999"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "bbottest.notreal"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
+ assert 0 == len([e for e in _graph_output_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
+
+ # sslcert with out-of-scope chain
+ events, all_events, all_events_nodups, graph_output_events, graph_output_batch_events = await do_scan(
+ "127.0.0.0/31",
+ modules=["speculate", "sslcert"],
+ whitelist=["127.0.1.0"],
+ _config={"dns_resolution": False, "scope_report_distance": 0, "internal_modules": {"speculate": {"ports": "9999"}}},
+ _dns_mock={("www.bbottest.notreal", "A"): "127.0.0.1", ("test.notreal", "A"): "127.0.1.0"},
+ )
+
+ assert len(events) == 3
+ assert 1 == len([e for e in events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 0 == len([e for e in events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1"])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999"])
+ assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
+ assert 0 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
+ assert 0 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
+
+ assert len(all_events) == 12
+ assert 1 == len([e for e in all_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 1])
+ assert 2 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 2 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal" and e.internal == True and e.scope_distance == 1 and str(e.module) == "speculate"])
+
+ assert len(all_events_nodups) == 10
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.0" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal" and e.internal == True and e.scope_distance == 2 and str(e.module) == "sslcert"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999" and e.internal == True and e.scope_distance == 0 and str(e.module) == "speculate"])
+ assert 1 == len([e for e in all_events_nodups if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal" and e.internal == True and e.scope_distance == 1 and str(e.module) == "speculate"])
+
+ for _graph_output_events in (graph_output_events, graph_output_batch_events):
+ assert len(_graph_output_events) == 5
+ assert 1 == len([e for e in graph_output_events if e.type == "IP_RANGE" and e.data == "127.0.0.0/31" and e.internal == False and e.scope_distance == 0])
+ assert 0 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.0"])
+ assert 1 == len([e for e in graph_output_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.1" and e.internal == True and e.scope_distance == 1])
+ assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.0:9999"])
+ assert 1 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "127.0.0.1:9999" and e.internal == True and e.scope_distance == 1])
+ assert 1 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "test.notreal" and e.internal == False and e.scope_distance == 0 and str(e.module) == "sslcert"])
+ assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME" and e.data == "www.bbottest.notreal"])
+ assert 0 == len([e for e in graph_output_events if e.type == "OPEN_TCP_PORT" and e.data == "test.notreal:9999"])
+ assert 0 == len([e for e in graph_output_events if e.type == "DNS_NAME_UNRESOLVED" and e.data == "notreal"])
+
+
+@pytest.mark.asyncio
+async def test_manager_blacklist(bbot_config, bbot_scanner, bbot_httpserver, caplog):
+
+ bbot_httpserver.expect_request(uri="/").respond_with_data(response_data="")
+
+ # dns search distance = 1, report distance = 0
+ config = {"dns_resolution": True, "scope_dns_search_distance": 1, "scope_report_distance": 0}
+ merged_config = OmegaConf.merge(bbot_config, OmegaConf.create(config))
+ scan = bbot_scanner(
+ "http://127.0.0.1:8888",
+ modules=["httpx", "excavate"],
+ config=merged_config,
+ whitelist=["127.0.0.0/29", "test.notreal"],
+ blacklist=["127.0.0.64/29"],
+ )
+ scan.helpers.dns.mock_dns({
+ ("www-prod.test.notreal", "A"): "127.0.0.66",
+ ("www-dev.test.notreal", "A"): "127.0.0.22",
+ })
+
+ events = [e async for e in scan.async_start()]
+
+ assert any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-dev.test.notreal:8888/"])
+ assert not any([e for e in events if e.type == "URL_UNVERIFIED" and e.data == "http://www-prod.test.notreal:8888/"])
+
+ assert 'Omitting due to blacklisted DNS associations: URL_UNVERIFIED("http://www-prod.test.notreal:8888/"' in caplog.text
+
+
+@pytest.mark.asyncio
+async def test_manager_scope_tagging(bbot_config, bbot_scanner):
+ scan = bbot_scanner("test.notreal", config=bbot_config)
+ e1 = scan.make_event("www.test.notreal", source=scan.root_event, tags=["affiliate"])
+ assert e1.scope_distance == 1
+ assert "distance-1" in e1.tags
+ assert "affiliate" in e1.tags
+
+ e2 = scan.make_event("dev.test.notreal", source=e1, tags=["affiliate"])
+ assert e2.scope_distance == 2
+ assert "affiliate" in e2.tags
+ assert "in-scope" not in e2.tags
+ distance_tags = [t for t in e2.tags if t.startswith("distance-")]
+ assert len(distance_tags) == 1
+ assert distance_tags[0] == "distance-2"
+
+ e2.scope_distance = 0
+ assert e2.scope_distance == 0
+ assert "in-scope" in e2.tags
+ assert "affiliate" not in e2.tags
+ distance_tags = [t for t in e2.tags if t.startswith("distance-")]
+ assert not distance_tags
diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index b4d61f516..6f8b8870f 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -24,7 +24,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h
assert base_output_module._event_precheck(localhost)[0] == True
localhost._internal = True
assert base_output_module._event_precheck(localhost)[0] == False
- localhost._force_output = True
+ localhost._internal = False
assert base_output_module._event_precheck(localhost)[0] == True
localhost._omit = True
assert base_output_module._event_precheck(localhost)[0] == False
@@ -33,7 +33,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h
for module_class in (BaseModule, BaseOutputModule, BaseReportModule, BaseInternalModule):
base_module = module_class(scan)
localhost2 = scan.make_event("127.0.0.2", source=events.subdomain)
- localhost2.set_scope_distance(0)
+ localhost2.scope_distance = 0
# base cases
base_module._watched_events = None
base_module.watched_events = ["*"]
@@ -54,7 +54,7 @@ async def test_modules_basic(scan, helpers, events, bbot_config, bbot_scanner, h
base_module.watched_events = ["IP_ADDRESS", "IP_RANGE"]
ip_range = scan.make_event("127.0.0.0/24", dummy=True)
localhost4 = scan.make_event("127.0.0.1", source=ip_range)
- localhost4.set_scope_distance(0)
+ localhost4.scope_distance = 0
localhost4.module = "plumbus"
assert base_module._event_precheck(localhost4)[0] == True
localhost4.module = "speculate"
@@ -190,11 +190,11 @@ async def test_modules_basic_perhostonly(scan, helpers, events, bbot_config, bbo
url_1 = per_host_scan.make_event(
"http://evilcorp.com/1", event_type="URL", source=per_host_scan.root_event, tags=["status-200"]
)
- url_1.set_scope_distance(0)
+ url_1.scope_distance = 0
url_2 = per_host_scan.make_event(
"http://evilcorp.com/2", event_type="URL", source=per_host_scan.root_event, tags=["status-200"]
)
- url_2.set_scope_distance(0)
+ url_2.scope_distance = 0
valid_1, reason_1 = await module._event_postcheck(url_1)
valid_2, reason_2 = await module._event_postcheck(url_2)
@@ -231,11 +231,11 @@ async def test_modules_basic_perdomainonly(scan, helpers, events, bbot_config, b
url_1 = per_domain_scan.make_event(
"http://www.evilcorp.com/1", event_type="URL", source=per_domain_scan.root_event, tags=["status-200"]
)
- url_1.set_scope_distance(0)
+ url_1.scope_distance = 0
url_2 = per_domain_scan.make_event(
"http://mail.evilcorp.com/2", event_type="URL", source=per_domain_scan.root_event, tags=["status-200"]
)
- url_2.set_scope_distance(0)
+ url_2.scope_distance = 0
valid_1, reason_1 = await module._event_postcheck(url_1)
valid_2, reason_2 = await module._event_postcheck(url_2)
diff --git a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
index b566d9f70..2f4303013 100644
--- a/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
+++ b/bbot/test/test_step_2/module_tests/test_module_asset_inventory.py
@@ -4,8 +4,8 @@
class TestAsset_Inventory(ModuleTestBase):
targets = ["127.0.0.1", "bbottest.notreal"]
scan_name = "asset_inventory_test"
- config_overrides = {"dns_resolution": True, "internal_modules": {"speculate": {"ports": "9999"}}}
- modules_overrides = ["asset_inventory", "speculate", "sslcert"]
+ config_overrides = {"dns_resolution": True, "internal_modules": {"nmap": {"ports": "9999"}}}
+ modules_overrides = ["asset_inventory", "nmap", "sslcert"]
async def setup_before_prep(self, module_test):
old_resolve_fn = module_test.scan.helpers.dns.resolve_event
diff --git a/bbot/test/test_step_2/module_tests/test_module_oauth.py b/bbot/test/test_step_2/module_tests/test_module_oauth.py
index c586fb419..eac2db2ac 100644
--- a/bbot/test/test_step_2/module_tests/test_module_oauth.py
+++ b/bbot/test/test_step_2/module_tests/test_module_oauth.py
@@ -227,4 +227,4 @@ def check(self, module_test, events):
== "Potentially Sprayable OAUTH Endpoint (domain: evilcorp.com) at https://evilcorp.okta.com/oauth2/v1/token"
for e in events
)
- assert any(e.data == "sts.windows.net" for e in events)
+ assert any(e.data == "https://sts.windows.net/cc74fc12-4142-400e-a653-f98bdeadbeef/" for e in events)
diff --git a/bbot/test/test_step_2/module_tests/test_module_sslcert.py b/bbot/test/test_step_2/module_tests/test_module_sslcert.py
index 51d9f8dcc..1a8796ab5 100644
--- a/bbot/test/test_step_2/module_tests/test_module_sslcert.py
+++ b/bbot/test/test_step_2/module_tests/test_module_sslcert.py
@@ -3,8 +3,17 @@
class TestSSLCert(ModuleTestBase):
targets = ["127.0.0.1:9999", "bbottest.notreal"]
+ config_overrides = {"scope_report_distance": 1}
def check(self, module_test, events):
- assert any(
- e.data == "www.bbottest.notreal" and str(e.module) == "sslcert" for e in events
- ), "Failed to detect subdomain"
+ assert len(events) == 6
+ assert 1 == len(
+ [
+ e
+ for e in events
+ if e.data == "www.bbottest.notreal" and str(e.module) == "sslcert" and e.scope_distance == 0
+ ]
+ ), "Failed to detect subject alternate name (SAN)"
+ assert 1 == len(
+ [e for e in events if e.data == "test.notreal" and str(e.module) == "sslcert" and e.scope_distance == 1]
+ ), "Failed to detect main subject"
diff --git a/bbot/test/testsslcert.pem b/bbot/test/testsslcert.pem
index 58795f01d..240034d21 100644
--- a/bbot/test/testsslcert.pem
+++ b/bbot/test/testsslcert.pem
@@ -1,32 +1,19 @@
-----BEGIN CERTIFICATE-----
-MIIFcTCCA1mgAwIBAgIUbj+lT6F+42NhnHW4I5nMg8f7fVYwDQYJKoZIhvcNAQEL
-BQAwRzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQKDANCTFMxHTAb
-BgNVBAMMFHd3dy5iYm90dGVzdC5ub3RyZWFsMCAXDTIzMDYxNTE1MjcxOVoYDzIw
-NTAxMDMwMTUyNzE5WjBHMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV
-BAoMA0JMUzEdMBsGA1UEAwwUd3d3LmJib3R0ZXN0Lm5vdHJlYWwwggIiMA0GCSqG
-SIb3DQEBAQUAA4ICDwAwggIKAoICAQC+Xown/v4Ow/kdGL2FHiF4wBBYZqwjXSFG
-W2ch2KHv2S9LKhfw+sesv0LAa1JEnBkA0BYoTS35H9W7PWeClA7fyeWwyv4BWoe6
-7aCldOdm6r3KtHswK1BYfZfi6zvPebTr08Bo2sUIw/C75XXUYX5aqAIjHwFzE7Fm
-/wadTgNAk8Qt16esrfoeJGP5by+rUraJa2aOsQZ20CCKtvMKc7aVev7XTYb8ZpGo
-a9b6qW4pn3+AeF8ZlzE7VUHVPObsSeZYEvQ1y+jEcISFtzkrPmsEs0mlatOdJzoJ
-CZCUgtkK4oqU2JgWH0dc7/slvuEolCJ3LgbiZ1MS7lq5CJSk/P09twn681/UTBav
-V9XR4x554Qta3DD4wE4HBhQ0HNtofQThlq9kQXyKbQimLFntyNI++mArLxakUCfP
-JwySG2VSppE3PS9Mv4/hF5134pkOcZ7vfUcxS941AQWs4LAqN/P16XDOkC97rsVm
-SwEoNlZm4saRRUeK6+YO1lKScD59Uv/XOTsnTqVU5EX/NNpCNFZkO+Loqzm/Cxg1
-XZoL4ggQHBq2hU6EGokRk7PGpPFnCwTevpL9FHFkYZgs0Q/gmxDSDz13bzpIegoA
-dSZ6qNgm69Uar5x7wxh6m7qgtVSlbT3mvhNRZAdfQjA0V1ei4AcjI+7ZZOsMr/AI
-qXDgXImzWwIDAQABo1MwUTAdBgNVHQ4EFgQU3mja6Q9QHOqYZusrPg7DFsKdKwIw
-HwYDVR0jBBgwFoAU3mja6Q9QHOqYZusrPg7DFsKdKwIwDwYDVR0TAQH/BAUwAwEB
-/zANBgkqhkiG9w0BAQsFAAOCAgEAY22H/Yn2ZHEe/OsoN4nkb2TlCSNN3hWz/6kj
-peivRuFzC3lIEThZq1QPi1ncLY0ZVFpbWtZxA2NCi6QN/nDJyjYnBzxkluUJWq15
-m4ktZNCDbvAXzTl5CFAzCpsPFITWusx10rD5UxV9qmmJ+ruaL/IlqKVYgcNlUam/
-tJjmMIOtmnszD4Z01v1M4MpLsaZXSGIqGiTAZNydbFF5bmccZV/e+8SBFv78Hoqh
-OEk8CCCLG31KF0q88YuNjA0B62g2B+xZRy3fxaZLm5mL56ZaMf6muezEI2vVUKPh
-0KWSnnxhNtLPNIB26zx6Epr6mUV9zccSKg9n9pZf7gPNWbQreV1tgFIoavI7XZvE
-dyOAwXyo1dnoyEmATanVLn/XutEyjHHtG/QnzV87/MoATVMeGwoteSoq1TGcXn1N
-xvmDt+VKrYSgyIGI3eFk1/WZXCQApL61SZItfK3kPEfKFPhWj2O7TtYT7LTeaAbO
-/n64t7fPeSB4d6zUL1lxhtPn5DwcZnEenc098N7yqtdNtjYJRPn3qTIqAI29d6UU
-puylfyyoLUwIXWXnOGnkJV5/DWBLbLRSWxMY8+YhIHaDv9VtAsPhCz4uKjLHk35c
-OztJlxpMzfM9VuttMzCD1aqufdLRtMGS9US+0ZrwQyjix3rkgovbFOOGrWYYu95A
-gWR5KGU=
+MIIDADCCAeigAwIBAgIUJnHoP2WYqS692n3bHQxkGlYlX1MwDQYJKoZIhvcNAQEL
+BQAwFzEVMBMGA1UEAwwMdGVzdC5ub3RyZWFsMCAXDTIzMTAxMzE3NTM0NFoYDzIw
+NTEwMjI3MTc1MzQ0WjAXMRUwEwYDVQQDDAx0ZXN0Lm5vdHJlYWwwggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDYDFf5yrTe23FF2zv2dQxQs+VdwxF7lCS/
+F6Tycuh/7+4aDLG9+3IQMeqFE7VlnaQb/M2QHsjMCeFlHUnd1jxXbmt+dWQ5Pxtz
+A8Vi0ypDDM6flHoT/f4CTVdDd1sc99ExBHApDAvRi6yyEnu0DxaZqzNTIRP2ijQq
+eHDTO4Hx+K/K/NvSCF05FnASS5EnOCx745lURtETatdAwa7HZADZ8NDgG9Dj8fa/
+uRq3FclBbbbmq9LWKTw3cAEXTz8+5N9F2/xSGk7NZpvIv5u15gtfbMfZcVADLSVe
+HR6NCfzgd/ZiHAx8CJf/ZStlMYksxZDSkb7wpdm9KeWNUpTjVknhAgMBAAGjQjBA
+MB8GA1UdEQQYMBaCFHd3dy5iYm90dGVzdC5ub3RyZWFsMB0GA1UdDgQWBBQC20kP
+Jq3PPZoWef0lV+c/ckbocjANBgkqhkiG9w0BAQsFAAOCAQEAzTLHR72bt2Bxc0bF
+aUQtumrX1rtuO3Cb2AiqKLPgb3nwnP5q+RZq991U1vMUFTiXUjplh86/Bh5IRJ8X
+1HUnMwTo6Co/77Ezx3Na2L62ajg2TpLo5YDOkIgMlOI63cGuk0ahelyxcsFVYdgA
+2/Jrh/xsybdKA5l1VG5jxzZ3s9d0Gd1wXpNe+bpwFR7gby52TkibPPviZ/CKF7NB
+7UdVj+SREXuSWH5NIicNQ71MJNE4CNNCOwy+yVoGY2E7WzqZNE+KZW5K5Sxp4Pnb
+Z9ZnCPq5m0RL7wBd+BhB2WxLVuvt0XdVS3H21cGuD/NR7r4OAsUNrf1nUwARNKPu
+BgZQhw==
-----END CERTIFICATE-----
diff --git a/bbot/test/testsslkey.pem b/bbot/test/testsslkey.pem
index 6e90e108b..c5f8dd3ad 100644
--- a/bbot/test/testsslkey.pem
+++ b/bbot/test/testsslkey.pem
@@ -1,52 +1,28 @@
-----BEGIN PRIVATE KEY-----
-MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC+Xown/v4Ow/kd
-GL2FHiF4wBBYZqwjXSFGW2ch2KHv2S9LKhfw+sesv0LAa1JEnBkA0BYoTS35H9W7
-PWeClA7fyeWwyv4BWoe67aCldOdm6r3KtHswK1BYfZfi6zvPebTr08Bo2sUIw/C7
-5XXUYX5aqAIjHwFzE7Fm/wadTgNAk8Qt16esrfoeJGP5by+rUraJa2aOsQZ20CCK
-tvMKc7aVev7XTYb8ZpGoa9b6qW4pn3+AeF8ZlzE7VUHVPObsSeZYEvQ1y+jEcISF
-tzkrPmsEs0mlatOdJzoJCZCUgtkK4oqU2JgWH0dc7/slvuEolCJ3LgbiZ1MS7lq5
-CJSk/P09twn681/UTBavV9XR4x554Qta3DD4wE4HBhQ0HNtofQThlq9kQXyKbQim
-LFntyNI++mArLxakUCfPJwySG2VSppE3PS9Mv4/hF5134pkOcZ7vfUcxS941AQWs
-4LAqN/P16XDOkC97rsVmSwEoNlZm4saRRUeK6+YO1lKScD59Uv/XOTsnTqVU5EX/
-NNpCNFZkO+Loqzm/Cxg1XZoL4ggQHBq2hU6EGokRk7PGpPFnCwTevpL9FHFkYZgs
-0Q/gmxDSDz13bzpIegoAdSZ6qNgm69Uar5x7wxh6m7qgtVSlbT3mvhNRZAdfQjA0
-V1ei4AcjI+7ZZOsMr/AIqXDgXImzWwIDAQABAoICAFH3ztxn2IeODUTggrX/J8Vg
-2YjeyI6/ilTLhWgW6X8nIkx4du+NY413muvUKgudtMxbTLcUqE9kzDf/dALO8mqc
-ebh6Zw05KEkuCSZE5GA1gfv2YzZ15/X6Ofx8cfffPS3vsqDTVdR03cJhxi7ovamW
-PrlVn9+nYpLpCzWPQio7ldRxT2K0VIZG//8GR1260xrvcizNf22IJwLjTFm7Qiox
-1+nnmVAzb9M3f8zVSDhG03UDH1ua9JlKF/sTapDiLvrMSnehcXunwP/8SPLcQOM4
-uWSuzygTHFOI2UaA4qhjq+z0J+QamrUjJZ32hudchUyQtmkkuSnlFI4oaixyPC1o
-KkwPyV/y+ugBuq/bIOTlHwTjg68kDLNRct+uL+GYq6MWUcDl3VnhK3471gSm6RBq
-VMoulKDQg7tyjWM1+TZgT5JRuCkXRw3TUm9RVrWMUa4VAPQCW+zgNf6TITI3IPMx
-R+SJ3SlGtkmU+kWIqMvFtGAMjttNuEpba9jnhtlO3O1WuZXcdn+oD1qRq2ifNIN3
-w2DZQ+waYFa5NO2a26KNxSA22S32AWLu7nA3zR8eomEFIzlrvNYUPo7g4kyeL2tA
-tmhSN6gRge5c0hymdtozXrNUad6rtV49oJDc1t47Wkqq2n205TEZyzxX/g2I0lPd
-vF9WVWva2T2OO0q+CJwRAoIBAQDJZ2BtjDIg16Q4oSARKfiN1e+74oIMnh3djf68
-fQXDYSZvs6UlTZdeFtJC0NppkaNyBUnjzgI1xtMCbLbYYT0kUTaikNU0Lvi5NjZH
-IQ+pTT21V7eKe0XMGhF8CLYt8Dh2JxCMPJDKhHwppOqIRRVsuUXEEYQlRQL/f/jm
-ONLMClwr+/a/DFYY/vqdt3al50dpszHB3POwKiMLpiANNir2k1fQ++OPW0Pm0c8/
-qe0Iuz7Qo629uBEWsv6MiEhpY7GfT8QBeqPKutdg1y5j7q6M1CgZSDClyhQ2bYRp
-5ZGib6I2Mdzf/vWFO+lTEfFWne7DLvN4ViW/8w31dc9ZUiErAoIBAQDx+WvS1LhQ
-rczCYh4em3L1k6G54dsnnIU7kudXsNIm/ND+YNbOJKxRm3jA0C6/Z7t+lSCKiAVK
-iG48Oz/c9wtWu1ck77fUY6IGbd+CM+o76KLmsadnOEaCrsWN8XJJBxSGxQBgD24F
-NjhxRwabzqAkpHcHT4LakrrWWoTcS1v059fxmbSWEmMxrHcQ5GwMmRU5p5d66Fwz
-mmT9J6W0yxitQRkO8vfu5C8m2UF/o7jYITY55JmNmTdp0z+p9U/gUVO6vo6/zROR
-DKLelvso42wTkgWNJFnxKnBUZIQ78PYdkL7Nf42tm0Rf/Dmti1SV5l/rMQ/4plGZ
-i0/QF4aXtb6RAoIBAAx0lTURJ4R6cZCC/m2xT8rNwT64WlyXZrdt8aQMg0+68VqV
-HJB7BPJvrN8u9WLz+8ywYQpykhZrTl9nA8PL97EISL2zAUxx3zXtEVxCwioxObC+
-VP5+cymVLyGhlEqxAXzlG5Pgpv+vQ9J7fXmh9Bo+Nv9SKf74DqBodKcgOoEd3udl
-dLQJ+kUfd7ZLQ04tSHBUAa7AkF2DrZkp0++dOvj20cZ93WvfNArRTOTeCGey1Mao
-aLVD7eKefTmJsBi8bp6wmymQQRLnjSxTAm5xs1/IOxadLU0De06GZVr7NI98IecC
-HGvJ47syolJd2LQHnl15yNrXUQxW5rUYbRNUDfMCggEAP4cSTdnXQic/GNGVwsdh
-GSRXU2v+k5m+SPO6lQUxR1ccr4xRGPH747kDrOt7OnlLemJAlt47NWQ4PMv5alPZ
-wD2TFXlyM+qsgS+bfJ2Qo4XiLtr5bloR+QFVkHyRWqbnNrLF9HJYmjmv+91/2KwY
-00XykhLH8gaP0vMU6plGXTf5M8784GGXdVUoNWEjFPOj5O0hEf2rJ0kjtmsQUDT6
-F5J9t7UGbsutrW9giNg9EVE24WApCEnviHMTxdVH0UbIysB4zdZTisNchz/wVL3Q
-hBrUJkgi7PQNZbq2FN6gXwa/cUhJtAiNnP2lTvDDlk9TY3YQEWHpDnzKyYFDrtTi
-YQKCAQA99iclLk7fWv+nNoEpAmI+qrtBtKNIQOvJ+0ACx61l5ycs5R9dpRVhaeB0
-vJMDm3r8ZouOYgycZDENk07bMhRtlFb2XKf10vnhTReQx9HJSsy/ZzhIJKFfTcbY
-SBH+VoxIX8U3/Da8cJ4VT+JM8rFHk94lsadl+wpRrwWaYgDCVcU3zwFIX0HBEUDi
-DZ93FdK/pVNdUsxQtm4v/ctreXmfwGTVnh8O92pDRw2vrqin7yTnElw42BNcGszW
-wCIuxiWtE3O10zj04o71MTtA5IN5l4xUPO/TpKjFqi2nfqJQOnlOqxgzVQAoUyqq
-TxNrvplI4d8A4JF6Q427dhu2M7vy
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDYDFf5yrTe23FF
+2zv2dQxQs+VdwxF7lCS/F6Tycuh/7+4aDLG9+3IQMeqFE7VlnaQb/M2QHsjMCeFl
+HUnd1jxXbmt+dWQ5PxtzA8Vi0ypDDM6flHoT/f4CTVdDd1sc99ExBHApDAvRi6yy
+Enu0DxaZqzNTIRP2ijQqeHDTO4Hx+K/K/NvSCF05FnASS5EnOCx745lURtETatdA
+wa7HZADZ8NDgG9Dj8fa/uRq3FclBbbbmq9LWKTw3cAEXTz8+5N9F2/xSGk7NZpvI
+v5u15gtfbMfZcVADLSVeHR6NCfzgd/ZiHAx8CJf/ZStlMYksxZDSkb7wpdm9KeWN
+UpTjVknhAgMBAAECggEAKsJYqB7LKN9YHhoLllXoo9FS+DlrDKEPm8V3dyewZd/L
+6VpxVDc/hj6G2qNBr9ShHgvs+FTra1yaQDupeq8Tvr8jJcJgnWbkzSDmME64StBu
+VY2akrnei8CYYIkvHn7ap3+oHiuc7DJfcdfwJT0mPTAxxoZhr9X/CJfRRrE8oPG4
+6w9WNS0CuyoDZ++xYwbWkNsF4XXtoOfkVgyXgtZDlIEAyRLvzVymDE05JjkmLRWt
+mmk4dmxJrYh/vd0DNAK3w1qmV3iaACs+1KG/TSNKeTrDipl+WyZE5KpJe/wPiOSV
+KG5hm4pXHRkN250k/5xUWWv+zQEC+fLd/JJdBev6JQKBgQDs8a28ltKstxYyTX2j
+W2L/C+jdQi1COCN7u3M0rFKw+vxFoTnvlCj1X9FLQ9lbgZr96vE2cpSMmskkmDDb
+KVneR6xBNLdqa/S0of/Ax9ZXwxR4k5EPUYMh3yfxuISbQyGe7qqLCeTbNZldqSYO
+igGpQ0nhxFwEJ7d9VzZrBTC7JQKBgQDpbHN8jlxzBQIox56wB+CS2ISj6c0dsIOI
+76J0nEdJ4qoDK1k185xaUvUyn43LmQOF2UHfwKOwVz09YwX+vFRkVNFKs7KCtUqg
+e0Z7C9oiSzeqKwxUope4yKz8MYtRFgUhAwrBa0WyRRLAyfRJQ6mPzie0GdVWt4pM
+tZ889lvVDQKBgQCCNo70BS7iG/vmyQ8ypxZQc4sVjTiyG4fkh69YUxteh4/79A6S
+yyl3L6Ela7QXxbIXuPW2pmFco/PGWJ0A1Ei/D0Rq0T27Dnj8i8qxdyEkOeEWIoKl
+mHYoNysMfArkCJCBd0fiAR30GhCemEaB1vXyvzfrCq5G2kzMZRFS3xdYwQKBgFl0
+sp2dgVijJryyI+KaYjpkuBCJXY5vQzmLfNrruXZbY4RrbHj8r4L+H/ISq6jHL05w
+gIpbrV+7T0DjXjzNuBnrV3ole9gT2lG+bLhjRmm2IdMZRFR7K2IppgHQiu+8XKLW
+I50Um1VCm3k+7FvXjngKLbUb4WKmXF4hjLE0SOVRAoGAZIXumUjy26y1dLL0E9F2
+HC5YEQfokVylQCWV+Ws5yjAZnAij8i0DWPHf2zvLAJ2BbmeMQAuYj5bUN1AUIFpG
+/ve/yLM0635dKgoH1Zlk83iQMrjXAuXkKc4gwfocPUnGFJ/LRXufodIKI2SP7Nff
+iVrq/w6VxVfc7EK0a7bfzxo=
-----END PRIVATE KEY-----
diff --git a/pyproject.toml b/pyproject.toml
index 395c60f49..35a8ecc1a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,6 +86,7 @@ build-backend = "poetry_dynamic_versioning.backend"
[tool.black]
line-length = 119
+extend-exclude = "(test_step_1/test_manager_*)"
[tool.poetry-dynamic-versioning]
enable = true