diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index ce627f695..d4a37b8f2 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -12,6 +12,7 @@ from copy import copy from pathlib import Path from typing import Optional +from zoneinfo import ZoneInfo from contextlib import suppress from radixtarget import RadixTarget from urllib.parse import urljoin, parse_qs @@ -40,6 +41,7 @@ validators, get_file_extension, ) +from bbot.models.helpers import utc_datetime_validator log = logging.getLogger("bbot.core.event") @@ -755,7 +757,7 @@ def __contains__(self, other): return bool(radixtarget.search(other.host)) return False - def json(self, mode="json", siem_friendly=False): + def json(self, mode="json"): """ Serializes the event object to a JSON-compatible dictionary. @@ -764,7 +766,6 @@ def json(self, mode="json", siem_friendly=False): Parameters: mode (str): Specifies the data serialization mode. Default is "json". Other options include "graph", "human", and "id". - siem_friendly (bool): Whether to format the JSON in a way that's friendly to SIEM ingestion by Elastic, Splunk, etc. This ensures the value of "data" is always the same type (a dictionary). Returns: dict: JSON-serializable dictionary representation of the event object. @@ -781,10 +782,12 @@ def json(self, mode="json", siem_friendly=False): data = data_attr else: data = smart_decode(self.data) - if siem_friendly: - j["data"] = {self.type: data} - else: + if isinstance(data, str): j["data"] = data + elif isinstance(data, dict): + j["data_json"] = data + else: + raise ValueError(f"Invalid data type: {type(data)}") # host, dns children if self.host: j["host"] = str(self.host) @@ -802,7 +805,7 @@ def json(self, mode="json", siem_friendly=False): if self.scan: j["scan"] = self.scan.id # timestamp - j["timestamp"] = self.timestamp.isoformat() + j["timestamp"] = utc_datetime_validator(self.timestamp).timestamp() # parent event parent_id = self.parent_id if parent_id: @@ -811,8 +814,7 @@ def json(self, mode="json", siem_friendly=False): if parent_uuid: j["parent_uuid"] = parent_uuid # tags - if self.tags: - j.update({"tags": list(self.tags)}) + j.update({"tags": list(self.tags)}) # parent module if self.module: j.update({"module": str(self.module)}) @@ -1728,7 +1730,7 @@ def make_event( ) -def event_from_json(j, siem_friendly=False): +def event_from_json(j): """ Creates an event object from a JSON dictionary. @@ -1760,10 +1762,12 @@ def event_from_json(j, siem_friendly=False): "context": j.get("discovery_context", None), "dummy": True, } - if siem_friendly: - data = j["data"][event_type] - else: - data = j["data"] + data = j.get("data_json", None) + if data is None: + data = j.get("data", None) + if data is None: + json_pretty = json.dumps(j, indent=2) + raise ValueError(f"data or data_json must be provided. JSON: {json_pretty}") kwargs["data"] = data event = make_event(**kwargs) event_uuid = j.get("uuid", None) @@ -1773,7 +1777,11 @@ def event_from_json(j, siem_friendly=False): resolved_hosts = j.get("resolved_hosts", []) event._resolved_hosts = set(resolved_hosts) - event.timestamp = datetime.datetime.fromisoformat(j["timestamp"]) + # accept both isoformat and unix timestamp + try: + event.timestamp = datetime.datetime.fromtimestamp(j["timestamp"], ZoneInfo("UTC")) + except Exception: + event.timestamp = datetime.datetime.fromisoformat(j["timestamp"]) event.scope_distance = j["scope_distance"] parent_id = j.get("parent", None) if parent_id is not None: diff --git a/bbot/models/helpers.py b/bbot/models/helpers.py new file mode 100644 index 000000000..b94bc976c --- /dev/null +++ b/bbot/models/helpers.py @@ -0,0 +1,20 @@ +from datetime import datetime +from zoneinfo import ZoneInfo + + +def utc_datetime_validator(d: datetime) -> datetime: + """ + Converts all dates into UTC + """ + if d.tzinfo is not None: + return d.astimezone(ZoneInfo("UTC")) + else: + return d.replace(tzinfo=ZoneInfo("UTC")) + + +def utc_now() -> datetime: + return datetime.now(ZoneInfo("UTC")) + + +def utc_now_timestamp() -> datetime: + return utc_now().timestamp() diff --git a/bbot/models/pydantic.py b/bbot/models/pydantic.py new file mode 100644 index 000000000..356ab2e44 --- /dev/null +++ b/bbot/models/pydantic.py @@ -0,0 +1,131 @@ +import logging +from pydantic import BaseModel, ConfigDict, Field +from typing import Optional, List, Union, Annotated + +from bbot.models.helpers import utc_now_timestamp + +log = logging.getLogger("bbot_server.models") + + +class BBOTBaseModel(BaseModel): + model_config = ConfigDict(extra="ignore") + + def __hash__(self): + return hash(self.to_json()) + + def __eq__(self, other): + return hash(self) == hash(other) + + @classmethod + def _indexed_fields(cls): + return sorted(field_name for field_name, field in cls.model_fields.items() if "indexed" in field.metadata) + + # we keep these because they were a lot of work to make and maybe someday they'll be useful again + + # @classmethod + # def _get_type_hints(cls): + # """ + # Drills down past all the Annotated, Optional, and Union layers to get the underlying type hint + # """ + # type_hints = get_type_hints(cls) + # unwrapped_type_hints = {} + # for field_name in cls.model_fields: + # type_hint = type_hints[field_name] + # while 1: + # if getattr(type_hint, "__origin__", None) in (Annotated, Optional, Union): + # type_hint = type_hint.__args__[0] + # else: + # break + # unwrapped_type_hints[field_name] = type_hint + # return unwrapped_type_hints + + # @classmethod + # def _datetime_fields(cls): + # datetime_fields = [] + # for field_name, type_hint in cls._get_type_hints().items(): + # if type_hint == datetime: + # datetime_fields.append(field_name) + # return sorted(datetime_fields) + + +### EVENT ### + + +class Event(BBOTBaseModel): + uuid: Annotated[str, "indexed", "unique"] + id: Annotated[str, "indexed"] + type: Annotated[str, "indexed"] + scope_description: str + data: Annotated[Optional[str], "indexed"] = None + data_json: Optional[dict] = None + host: Annotated[Optional[str], "indexed"] = None + port: Optional[int] = None + netloc: Optional[str] = None + # we store the host in reverse to allow for instant subdomain queries + # this works because indexes are left-anchored, but we need to search starting from the right side + reverse_host: Annotated[Optional[str], "indexed"] = "" + resolved_hosts: Union[List, None] = None + dns_children: Union[dict, None] = None + web_spider_distance: int = 10 + scope_distance: int = 10 + scan: Annotated[str, "indexed"] + timestamp: Annotated[float, "indexed"] + inserted_at: Annotated[Optional[float], "indexed"] = Field(default_factory=utc_now_timestamp) + parent: Annotated[str, "indexed"] + parent_uuid: Annotated[str, "indexed"] + tags: List = [] + module: Annotated[Optional[str], "indexed"] = None + module_sequence: Optional[str] = None + discovery_context: str = "" + discovery_path: List[str] = [] + parent_chain: List[str] = [] + + def __init__(self, **data): + super().__init__(**data) + if self.host: + self.reverse_host = self.host[::-1] + + def get_data(self): + if self.data is not None: + return self.data + return self.data_json + + +### SCAN ### + + +class Scan(BBOTBaseModel): + id: Annotated[str, "indexed", "unique"] + name: str + status: Annotated[str, "indexed"] + started_at: Annotated[float, "indexed"] + finished_at: Annotated[Optional[float], "indexed"] = None + duration_seconds: Optional[float] = None + duration: Optional[str] = None + target: dict + preset: dict + + @classmethod + def from_scan(cls, scan): + return cls( + id=scan.id, + name=scan.name, + status=scan.status, + started_at=scan.started_at, + ) + + +### TARGET ### + + +class Target(BBOTBaseModel): + name: str = "Default Target" + strict_scope: bool = False + seeds: List = [] + whitelist: List = [] + blacklist: List = [] + hash: Annotated[str, "indexed", "unique"] + scope_hash: Annotated[str, "indexed"] + seed_hash: Annotated[str, "indexed"] + whitelist_hash: Annotated[str, "indexed"] + blacklist_hash: Annotated[str, "indexed"] diff --git a/bbot/db/sql/models.py b/bbot/models/sql.py similarity index 83% rename from bbot/db/sql/models.py rename to bbot/models/sql.py index e937fad1e..8e3e059b0 100644 --- a/bbot/db/sql/models.py +++ b/bbot/models/sql.py @@ -3,13 +3,15 @@ import json import logging +from datetime import datetime from pydantic import ConfigDict from typing import List, Optional -from datetime import datetime, timezone from typing_extensions import Annotated from pydantic.functional_validators import AfterValidator from sqlmodel import inspect, Column, Field, SQLModel, JSON, String, DateTime as SQLADateTime +from bbot.models.helpers import utc_now_timestamp + log = logging.getLogger("bbot_server.models") @@ -27,14 +29,6 @@ def naive_datetime_validator(d: datetime): NaiveUTC = Annotated[datetime, AfterValidator(naive_datetime_validator)] -class CustomJSONEncoder(json.JSONEncoder): - def default(self, obj): - # handle datetime - if isinstance(obj, datetime): - return obj.isoformat() - return super().default(obj) - - class BBOTBaseModel(SQLModel): model_config = ConfigDict(extra="ignore") @@ -52,7 +46,7 @@ def validated(self): return self def to_json(self, **kwargs): - return json.dumps(self.validated.model_dump(), sort_keys=True, cls=CustomJSONEncoder, **kwargs) + return json.dumps(self.validated.model_dump(), sort_keys=True, **kwargs) @classmethod def _pk_column_names(cls): @@ -72,20 +66,13 @@ class Event(BBOTBaseModel, table=True): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - data = self._get_data(self.data, self.type) - self.data = {self.type: data} if self.host: self.reverse_host = self.host[::-1] def get_data(self): - return self._get_data(self.data, self.type) - - @staticmethod - def _get_data(data, type): - # handle SIEM-friendly format - if isinstance(data, dict) and list(data) == [type]: - return data[type] - return data + if self.data is not None: + return self.data + return self.data_json uuid: str = Field( primary_key=True, @@ -94,11 +81,12 @@ def _get_data(data, type): ) id: str = Field(index=True) type: str = Field(index=True) - scope_description: str - data: dict = Field(sa_type=JSON) + data: Optional[str] = Field(default=None, index=True) + data_json: Optional[dict] = Field(default=None, sa_type=JSON) host: Optional[str] port: Optional[int] netloc: Optional[str] + scope_description: str # store the host in reversed form for efficient lookups by domain reverse_host: Optional[str] = Field(default="", exclude=True, index=True) resolved_hosts: List = Field(default=[], sa_type=JSON) @@ -106,7 +94,8 @@ def _get_data(data, type): web_spider_distance: int = 10 scope_distance: int = Field(default=10, index=True) scan: str = Field(index=True) - timestamp: NaiveUTC = Field(index=True) + timestamp: float = Field(index=True) + inserted_at: float = Field(default_factory=utc_now_timestamp) parent: str = Field(index=True) tags: List = Field(default=[], sa_type=JSON) module: str = Field(index=True) @@ -114,7 +103,6 @@ def _get_data(data, type): discovery_context: str = "" discovery_path: List[str] = Field(default=[], sa_type=JSON) parent_chain: List[str] = Field(default=[], sa_type=JSON) - inserted_at: NaiveUTC = Field(default_factory=lambda: datetime.now(timezone.utc)) ### SCAN ### diff --git a/bbot/modules/output/elastic.py b/bbot/modules/output/elastic.py new file mode 100644 index 000000000..15bc023df --- /dev/null +++ b/bbot/modules/output/elastic.py @@ -0,0 +1,22 @@ +from .http import HTTP + + +class Elastic(HTTP): + watched_events = ["*"] + metadata = { + "description": "Send scan results to Elasticsearch", + "created_date": "2022-11-21", + "author": "@TheTechromancer", + } + options = { + "url": "", + "username": "elastic", + "password": "changeme", + "timeout": 10, + } + options_desc = { + "url": "Elastic URL (e.g. https://localhost:9200//_doc)", + "username": "Elastic username", + "password": "Elastic password", + "timeout": "HTTP timeout", + } diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 9d9241da0..0af65a87d 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,3 +1,4 @@ +from bbot.models.pydantic import Event from bbot.modules.output.base import BaseOutputModule @@ -15,7 +16,6 @@ class HTTP(BaseOutputModule): "username": "", "password": "", "timeout": 10, - "siem_friendly": False, } options_desc = { "url": "Web URL", @@ -24,14 +24,12 @@ class HTTP(BaseOutputModule): "username": "Username (basic auth)", "password": "Password (basic auth)", "timeout": "HTTP timeout", - "siem_friendly": "Format JSON in a SIEM-friendly way for ingestion into Elastic, Splunk, etc.", } async def setup(self): self.url = self.config.get("url", "") self.method = self.config.get("method", "POST") self.timeout = self.config.get("timeout", 10) - self.siem_friendly = self.config.get("siem_friendly", False) self.headers = {} bearer = self.config.get("bearer", "") if bearer: @@ -51,12 +49,15 @@ async def setup(self): async def handle_event(self, event): while 1: + event_json = event.json() + event_pydantic = Event(**event_json) + event_json = event_pydantic.model_dump(exclude_none=True) response = await self.helpers.request( url=self.url, method=self.method, auth=self.auth, headers=self.headers, - json=event.json(siem_friendly=self.siem_friendly), + json=event_json, ) is_success = False if response is None else response.is_success if not is_success: diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index a35fa6aed..b93d1e4e3 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -11,20 +11,18 @@ class JSON(BaseOutputModule): "created_date": "2022-04-07", "author": "@TheTechromancer", } - options = {"output_file": "", "siem_friendly": False} + options = {"output_file": ""} options_desc = { "output_file": "Output to file", - "siem_friendly": "Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc.", } _preserve_graph = True async def setup(self): self._prep_output_dir("output.json") - self.siem_friendly = self.config.get("siem_friendly", False) return True async def handle_event(self, event): - event_json = event.json(siem_friendly=self.siem_friendly) + event_json = event.json() event_str = json.dumps(event_json) if self.file is not None: self.file.write(event_str + "\n") diff --git a/bbot/modules/output/mongo.py b/bbot/modules/output/mongo.py new file mode 100644 index 000000000..6ad16620f --- /dev/null +++ b/bbot/modules/output/mongo.py @@ -0,0 +1,81 @@ +from motor.motor_asyncio import AsyncIOMotorClient + +from bbot.models.pydantic import Event, Scan, Target +from bbot.modules.output.base import BaseOutputModule + + +class Mongo(BaseOutputModule): + watched_events = ["*"] + meta = { + "description": "Output scan data to a MongoDB database", + "created_date": "2024-11-17", + "author": "@TheTechromancer", + } + options = { + "uri": "mongodb://localhost:27017", + "database": "bbot", + "username": "", + "password": "", + "collection_prefix": "", + } + options_desc = { + "uri": "The URI of the MongoDB server", + "database": "The name of the database to use", + "username": "The username to use to connect to the database", + "password": "The password to use to connect to the database", + "collection_prefix": "Prefix the name of each collection with this string", + } + deps_pip = ["motor~=3.6.0"] + + async def setup(self): + self.uri = self.config.get("uri", "mongodb://localhost:27017") + self.username = self.config.get("username", "") + self.password = self.config.get("password", "") + self.db_client = AsyncIOMotorClient(self.uri, username=self.username, password=self.password) + + # Ping the server to confirm a successful connection + try: + await self.db_client.admin.command("ping") + self.verbose("MongoDB connection successful") + except Exception as e: + return False, f"Failed to connect to MongoDB: {e}" + + self.db_name = self.config.get("database", "bbot") + self.db = self.db_client[self.db_name] + self.collection_prefix = self.config.get("collection_prefix", "") + self.events_collection = self.db[f"{self.collection_prefix}events"] + self.scans_collection = self.db[f"{self.collection_prefix}scans"] + self.targets_collection = self.db[f"{self.collection_prefix}targets"] + + # Build an index for each field in reverse_host and host + for field_name, field in Event.model_fields.items(): + if "indexed" in field.metadata: + unique = "unique" in field.metadata + await self.events_collection.create_index([(field_name, 1)], unique=unique) + self.verbose(f"Index created for field: {field_name} (unique={unique})") + + return True + + async def handle_event(self, event): + event_json = event.json() + event_pydantic = Event(**event_json) + await self.events_collection.insert_one(event_pydantic.model_dump()) + + if event.type == "SCAN": + scan_json = Scan(**event.data_json).model_dump() + existing_scan = await self.scans_collection.find_one({"id": event_pydantic.id}) + if existing_scan: + await self.scans_collection.replace_one({"id": event_pydantic.id}, scan_json) + self.verbose(f"Updated scan event with ID: {event_pydantic.id}") + else: + # Insert as a new scan if no existing scan is found + await self.scans_collection.insert_one(event_pydantic.model_dump()) + self.verbose(f"Inserted new scan event with ID: {event_pydantic.id}") + + target_data = scan_json.get("target", {}) + target = Target(**target_data) + existing_target = await self.targets_collection.find_one({"hash": target.hash}) + if existing_target: + await self.targets_collection.replace_one({"hash": target.hash}, target.model_dump()) + else: + await self.targets_collection.insert_one(target.model_dump()) diff --git a/bbot/modules/templates/sql.py b/bbot/modules/templates/sql.py index 39b4e6f00..42f549455 100644 --- a/bbot/modules/templates/sql.py +++ b/bbot/modules/templates/sql.py @@ -3,7 +3,7 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession -from bbot.db.sql.models import Event, Scan, Target +from bbot.models.sql import Event, Scan, Target from bbot.modules.output.base import BaseOutputModule diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 49114a5b5..0915c4cb9 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -7,6 +7,7 @@ from pathlib import Path from sys import exc_info from datetime import datetime +from zoneinfo import ZoneInfo from collections import OrderedDict from bbot import __version__ @@ -327,8 +328,8 @@ async def async_start_without_generator(self): async def async_start(self): """ """ - self.start_time = datetime.now() - self.root_event.data["started_at"] = self.start_time.isoformat() + self.start_time = datetime.now(ZoneInfo("UTC")) + self.root_event.data["started_at"] = self.start_time.timestamp() try: await self._prep() @@ -436,7 +437,7 @@ async def _mark_finished(self): else: status = "FINISHED" - self.end_time = datetime.now() + self.end_time = datetime.now(ZoneInfo("UTC")) self.duration = self.end_time - self.start_time self.duration_seconds = self.duration.total_seconds() self.duration_human = self.helpers.human_timedelta(self.duration) @@ -500,7 +501,8 @@ async def setup_modules(self, remove_failed=True): self.modules[module.name].set_error_state() hard_failed.append(module.name) else: - self.info(f"Setup soft-failed for {module.name}: {msg}") + log_fn = self.warning if module._type == "output" else self.info + log_fn(f"Setup soft-failed for {module.name}: {msg}") soft_failed.append(module.name) if (not status) and (module._intercept or remove_failed): # if a intercept module fails setup, we always remove it @@ -1129,9 +1131,9 @@ def json(self): j["target"] = self.preset.target.json j["preset"] = self.preset.to_dict(redact_secrets=True) if self.start_time is not None: - j["started_at"] = self.start_time.isoformat() + j["started_at"] = self.start_time.timestamp() if self.end_time is not None: - j["finished_at"] = self.end_time.isoformat() + j["finished_at"] = self.end_time.timestamp() if self.duration is not None: j["duration_seconds"] = self.duration_seconds if self.duration_human is not None: diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index e1e3aa1b8..4d73d036c 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -147,48 +147,78 @@ def helpers(scan): @pytest.fixture def events(scan): + + dummy_module = scan._make_dummy_module("dummy_module") + class bbot_events: - localhost = scan.make_event("127.0.0.1", parent=scan.root_event) - ipv4 = scan.make_event("8.8.8.8", parent=scan.root_event) - netv4 = scan.make_event("8.8.8.8/30", parent=scan.root_event) - ipv6 = scan.make_event("2001:4860:4860::8888", parent=scan.root_event) - netv6 = scan.make_event("2001:4860:4860::8888/126", parent=scan.root_event) - domain = scan.make_event("publicAPIs.org", parent=scan.root_event) - subdomain = scan.make_event("api.publicAPIs.org", parent=scan.root_event) - email = scan.make_event("bob@evilcorp.co.uk", "EMAIL_ADDRESS", parent=scan.root_event) - open_port = scan.make_event("api.publicAPIs.org:443", parent=scan.root_event) + localhost = scan.make_event("127.0.0.1", parent=scan.root_event, module=dummy_module) + ipv4 = scan.make_event("8.8.8.8", parent=scan.root_event, module=dummy_module) + netv4 = scan.make_event("8.8.8.8/30", parent=scan.root_event, module=dummy_module) + ipv6 = scan.make_event("2001:4860:4860::8888", parent=scan.root_event, module=dummy_module) + netv6 = scan.make_event("2001:4860:4860::8888/126", parent=scan.root_event, module=dummy_module) + domain = scan.make_event("publicAPIs.org", parent=scan.root_event, module=dummy_module) + subdomain = scan.make_event("api.publicAPIs.org", parent=scan.root_event, module=dummy_module) + email = scan.make_event("bob@evilcorp.co.uk", "EMAIL_ADDRESS", parent=scan.root_event, module=dummy_module) + open_port = scan.make_event("api.publicAPIs.org:443", parent=scan.root_event, module=dummy_module) protocol = scan.make_event( - {"host": "api.publicAPIs.org", "port": 443, "protocol": "HTTP"}, "PROTOCOL", parent=scan.root_event + {"host": "api.publicAPIs.org", "port": 443, "protocol": "HTTP"}, + "PROTOCOL", + parent=scan.root_event, + module=dummy_module, + ) + ipv4_open_port = scan.make_event("8.8.8.8:443", parent=scan.root_event, module=dummy_module) + ipv6_open_port = scan.make_event( + "[2001:4860:4860::8888]:443", "OPEN_TCP_PORT", parent=scan.root_event, module=dummy_module + ) + url_unverified = scan.make_event( + "https://api.publicAPIs.org:443/hellofriend", parent=scan.root_event, module=dummy_module + ) + ipv4_url_unverified = scan.make_event( + "https://8.8.8.8:443/hellofriend", parent=scan.root_event, module=dummy_module + ) + ipv6_url_unverified = scan.make_event( + "https://[2001:4860:4860::8888]:443/hellofriend", parent=scan.root_event, module=dummy_module ) - ipv4_open_port = scan.make_event("8.8.8.8:443", parent=scan.root_event) - ipv6_open_port = scan.make_event("[2001:4860:4860::8888]:443", "OPEN_TCP_PORT", parent=scan.root_event) - url_unverified = scan.make_event("https://api.publicAPIs.org:443/hellofriend", parent=scan.root_event) - ipv4_url_unverified = scan.make_event("https://8.8.8.8:443/hellofriend", parent=scan.root_event) - ipv6_url_unverified = scan.make_event("https://[2001:4860:4860::8888]:443/hellofriend", parent=scan.root_event) url = scan.make_event( - "https://api.publicAPIs.org:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event + "https://api.publicAPIs.org:443/hellofriend", + "URL", + tags=["status-200"], + parent=scan.root_event, + module=dummy_module, ) ipv4_url = scan.make_event( - "https://8.8.8.8:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event + "https://8.8.8.8:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event, module=dummy_module ) ipv6_url = scan.make_event( - "https://[2001:4860:4860::8888]:443/hellofriend", "URL", tags=["status-200"], parent=scan.root_event + "https://[2001:4860:4860::8888]:443/hellofriend", + "URL", + tags=["status-200"], + parent=scan.root_event, + module=dummy_module, + ) + url_hint = scan.make_event( + "https://api.publicAPIs.org:443/hello.ash", "URL_HINT", parent=url, module=dummy_module ) - url_hint = scan.make_event("https://api.publicAPIs.org:443/hello.ash", "URL_HINT", parent=url) vulnerability = scan.make_event( {"host": "evilcorp.com", "severity": "INFO", "description": "asdf"}, "VULNERABILITY", parent=scan.root_event, + module=dummy_module, + ) + finding = scan.make_event( + {"host": "evilcorp.com", "description": "asdf"}, "FINDING", parent=scan.root_event, module=dummy_module + ) + vhost = scan.make_event( + {"host": "evilcorp.com", "vhost": "www.evilcorp.com"}, "VHOST", parent=scan.root_event, module=dummy_module ) - finding = scan.make_event({"host": "evilcorp.com", "description": "asdf"}, "FINDING", parent=scan.root_event) - vhost = scan.make_event({"host": "evilcorp.com", "vhost": "www.evilcorp.com"}, "VHOST", parent=scan.root_event) - http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event) + http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event, module=dummy_module) storage_bucket = scan.make_event( {"name": "storage", "url": "https://storage.blob.core.windows.net"}, "STORAGE_BUCKET", parent=scan.root_event, + module=dummy_module, ) - emoji = scan.make_event("💩", "WHERE_IS_YOUR_GOD_NOW", parent=scan.root_event) + emoji = scan.make_event("💩", "WHERE_IS_YOUR_GOD_NOW", parent=scan.root_event, module=dummy_module) bbot_events.all = [ # noqa: F841 bbot_events.localhost, diff --git a/bbot/test/test_step_1/test_bbot_fastapi.py b/bbot/test/test_step_1/test_bbot_fastapi.py index bad402071..617f95abb 100644 --- a/bbot/test/test_step_1/test_bbot_fastapi.py +++ b/bbot/test/test_step_1/test_bbot_fastapi.py @@ -28,7 +28,7 @@ def test_bbot_multiprocess(bbot_httpserver): assert len(events) >= 3 scan_events = [e for e in events if e["type"] == "SCAN"] assert len(scan_events) == 2 - assert any([e["data"] == "test@blacklanternsecurity.com" for e in events]) + assert any([e.get("data", "") == "test@blacklanternsecurity.com" for e in events]) def test_bbot_fastapi(bbot_httpserver): @@ -61,7 +61,7 @@ def test_bbot_fastapi(bbot_httpserver): assert len(events) >= 3 scan_events = [e for e in events if e["type"] == "SCAN"] assert len(scan_events) == 2 - assert any([e["data"] == "test@blacklanternsecurity.com" for e in events]) + assert any([e.get("data", "") == "test@blacklanternsecurity.com" for e in events]) finally: with suppress(Exception): diff --git a/bbot/test/test_step_1/test_db_models.py b/bbot/test/test_step_1/test_db_models.py new file mode 100644 index 000000000..9c7139069 --- /dev/null +++ b/bbot/test/test_step_1/test_db_models.py @@ -0,0 +1,59 @@ +from datetime import datetime +from zoneinfo import ZoneInfo + +from bbot.models.pydantic import Event +from bbot.core.event.base import BaseEvent +from bbot.models.helpers import utc_datetime_validator +from ..bbot_fixtures import * # noqa + + +def test_pydantic_models(events): + + # test datetime helpers + now = datetime.now(ZoneInfo("America/New_York")) + utc_now = utc_datetime_validator(now) + assert now.timestamp() == utc_now.timestamp() + now2 = datetime.fromtimestamp(utc_now.timestamp(), ZoneInfo("UTC")) + assert now2.timestamp() == utc_now.timestamp() + utc_now2 = utc_datetime_validator(now2) + assert utc_now2.timestamp() == utc_now.timestamp() + + test_event = Event(**events.ipv4.json()) + assert sorted(test_event._indexed_fields()) == [ + "data", + "host", + "id", + "inserted_at", + "module", + "parent", + "parent_uuid", + "reverse_host", + "scan", + "timestamp", + "type", + "uuid", + ] + + # convert events to pydantic and back, making sure they're exactly the same + for event in ("ipv4", "http_response", "finding", "vulnerability", "storage_bucket"): + e = getattr(events, event) + event_json = e.json() + event_pydantic = Event(**event_json) + event_pydantic_dict = event_pydantic.model_dump() + event_reconstituted = BaseEvent.from_json(event_pydantic.model_dump(exclude_none=True)) + assert isinstance(event_json["timestamp"], float) + assert isinstance(e.timestamp, datetime) + assert isinstance(event_pydantic.timestamp, float) + assert not "inserted_at" in event_json + assert isinstance(event_pydantic_dict["timestamp"], float) + assert isinstance(event_pydantic_dict["inserted_at"], float) + + event_pydantic_dict = event_pydantic.model_dump(exclude_none=True, exclude=["reverse_host", "inserted_at"]) + assert event_pydantic_dict == event_json + event_pydantic_dict.pop("scan") + event_pydantic_dict.pop("module") + event_pydantic_dict.pop("module_sequence") + assert event_reconstituted.json() == event_pydantic_dict + + +# TODO: SQL diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 8156fc796..faadbdaae 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -494,7 +494,7 @@ async def test_events(events, helpers): assert db_event.parent_chain[0] == str(db_event.uuid) assert db_event.parent.uuid == scan.root_event.uuid assert db_event.parent_uuid == scan.root_event.uuid - timestamp = db_event.timestamp.isoformat() + timestamp = db_event.timestamp.timestamp() json_event = db_event.json() assert isinstance(json_event["uuid"], str) assert json_event["uuid"] == str(db_event.uuid) @@ -515,7 +515,7 @@ async def test_events(events, helpers): assert reconstituted_event.uuid == db_event.uuid assert reconstituted_event.parent_uuid == scan.root_event.uuid assert reconstituted_event.scope_distance == 1 - assert reconstituted_event.timestamp.isoformat() == timestamp + assert reconstituted_event.timestamp.timestamp() == timestamp assert reconstituted_event.data == "evilcorp.com:80" assert reconstituted_event.type == "OPEN_TCP_PORT" assert reconstituted_event.host == "evilcorp.com" @@ -529,28 +529,17 @@ async def test_events(events, helpers): assert hostless_event_json["data"] == "asdf" assert not "host" in hostless_event_json - # SIEM-friendly serialize/deserialize - json_event_siemfriendly = db_event.json(siem_friendly=True) - assert json_event_siemfriendly["scope_distance"] == 1 - assert json_event_siemfriendly["data"] == {"OPEN_TCP_PORT": "evilcorp.com:80"} - assert json_event_siemfriendly["type"] == "OPEN_TCP_PORT" - assert json_event_siemfriendly["host"] == "evilcorp.com" - assert json_event_siemfriendly["timestamp"] == timestamp - reconstituted_event2 = event_from_json(json_event_siemfriendly, siem_friendly=True) - assert reconstituted_event2.scope_distance == 1 - assert reconstituted_event2.timestamp.isoformat() == timestamp - assert reconstituted_event2.data == "evilcorp.com:80" - assert reconstituted_event2.type == "OPEN_TCP_PORT" - assert reconstituted_event2.host == "evilcorp.com" - assert "127.0.0.1" in reconstituted_event2.resolved_hosts - http_response = scan.make_event(httpx_response, "HTTP_RESPONSE", parent=scan.root_event) assert http_response.parent_id == scan.root_event.id assert http_response.data["input"] == "http://example.com:80" json_event = http_response.json(mode="graph") + assert "data" in json_event + assert "data_json" not in json_event assert isinstance(json_event["data"], str) json_event = http_response.json() - assert isinstance(json_event["data"], dict) + assert "data" not in json_event + assert "data_json" in json_event + assert isinstance(json_event["data_json"], dict) assert json_event["type"] == "HTTP_RESPONSE" assert json_event["host"] == "example.com" assert json_event["parent"] == scan.root_event.id diff --git a/bbot/test/test_step_1/test_scan.py b/bbot/test/test_step_1/test_scan.py index f5f845826..5514571fa 100644 --- a/bbot/test/test_step_1/test_scan.py +++ b/bbot/test/test_step_1/test_scan.py @@ -144,7 +144,7 @@ async def test_python_output_matches_json(bbot_scanner): assert len(events) == 5 scan_events = [e for e in events if e["type"] == "SCAN"] assert len(scan_events) == 2 - assert all([isinstance(e["data"]["status"], str) for e in scan_events]) + assert all([isinstance(e["data_json"]["status"], str) for e in scan_events]) assert len([e for e in events if e["type"] == "DNS_NAME"]) == 1 assert len([e for e in events if e["type"] == "ORG_STUB"]) == 1 assert len([e for e in events if e["type"] == "IP_ADDRESS"]) == 1 diff --git a/bbot/test/test_step_2/module_tests/test_module_elastic.py b/bbot/test/test_step_2/module_tests/test_module_elastic.py new file mode 100644 index 000000000..710c22e0f --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_elastic.py @@ -0,0 +1,130 @@ +import time +import httpx +import asyncio + +from .base import ModuleTestBase + + +class TestElastic(ModuleTestBase): + config_overrides = { + "modules": { + "elastic": { + "url": "https://localhost:9200/bbot_test_events/_doc", + "username": "elastic", + "password": "bbotislife", + } + } + } + skip_distro_tests = True + + async def setup_before_prep(self, module_test): + # Start Elasticsearch container + await asyncio.create_subprocess_exec( + "docker", + "run", + "--name", + "bbot-test-elastic", + "--rm", + "-e", + "ELASTIC_PASSWORD=bbotislife", + "-e", + "cluster.routing.allocation.disk.watermark.low=96%", + "-e", + "cluster.routing.allocation.disk.watermark.high=97%", + "-e", + "cluster.routing.allocation.disk.watermark.flood_stage=98%", + "-p", + "9200:9200", + "-d", + "docker.elastic.co/elasticsearch/elasticsearch:8.16.0", + ) + + # Connect to Elasticsearch with retry logic + async with httpx.AsyncClient(verify=False) as client: + while True: + try: + # Attempt a simple operation to confirm the connection + response = await client.get("https://localhost:9200/_cat/health", auth=("elastic", "bbotislife")) + response.raise_for_status() + break + except Exception as e: + print(f"Connection failed: {e}. Retrying...", flush=True) + time.sleep(0.5) + + # Ensure the index is empty + await client.delete(f"https://localhost:9200/bbot_test_events", auth=("elastic", "bbotislife")) + print("Elasticsearch index cleaned up", flush=True) + + async def check(self, module_test, events): + try: + from bbot.models.pydantic import Event + + events_json = [e.json() for e in events] + events_json.sort(key=lambda x: x["timestamp"]) + + # Connect to Elasticsearch + async with httpx.AsyncClient(verify=False) as client: + + # refresh the index + await client.post(f"https://localhost:9200/bbot_test_events/_refresh", auth=("elastic", "bbotislife")) + + # Fetch all events from the index + response = await client.get( + f"https://localhost:9200/bbot_test_events/_search?size=100", auth=("elastic", "bbotislife") + ) + response_json = response.json() + import json + + print(f"response: {json.dumps(response_json, indent=2)}") + db_events = [hit["_source"] for hit in response_json["hits"]["hits"]] + + # make sure we have the same number of events + assert len(events_json) == len(db_events) + + for db_event in db_events: + assert isinstance(db_event["timestamp"], float) + assert isinstance(db_event["inserted_at"], float) + + # Convert to Pydantic objects and dump them + db_events_pydantic = [Event(**e).model_dump(exclude_none=True) for e in db_events] + db_events_pydantic.sort(key=lambda x: x["timestamp"]) + + # Find the main event with type DNS_NAME and data blacklanternsecurity.com + main_event = next( + ( + e + for e in db_events_pydantic + if e.get("type") == "DNS_NAME" and e.get("data") == "blacklanternsecurity.com" + ), + None, + ) + assert ( + main_event is not None + ), "Main event with type DNS_NAME and data blacklanternsecurity.com not found" + + # Ensure it has the reverse_host attribute + expected_reverse_host = "blacklanternsecurity.com"[::-1] + assert ( + main_event.get("reverse_host") == expected_reverse_host + ), f"reverse_host attribute is not correct, expected {expected_reverse_host}" + + # Events don't match exactly because the elastic ones have reverse_host and inserted_at + assert events_json != db_events_pydantic + for db_event in db_events_pydantic: + db_event.pop("reverse_host") + db_event.pop("inserted_at") + # They should match after removing reverse_host + assert events_json == db_events_pydantic, "Events do not match" + + finally: + # Clean up: Delete all documents in the index + async with httpx.AsyncClient(verify=False) as client: + response = await client.delete( + f"https://localhost:9200/bbot_test_events", + auth=("elastic", "bbotislife"), + params={"ignore": "400,404"}, + ) + print(f"Deleted documents from index", flush=True) + await asyncio.create_subprocess_exec( + "docker", "stop", "bbot-test-elastic", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) diff --git a/bbot/test/test_step_2/module_tests/test_module_http.py b/bbot/test/test_step_2/module_tests/test_module_http.py index 43b7189ad..d63476542 100644 --- a/bbot/test/test_step_2/module_tests/test_module_http.py +++ b/bbot/test/test_step_2/module_tests/test_module_http.py @@ -52,12 +52,3 @@ def check(self, module_test, events): assert self.headers_correct == True assert self.method_correct == True assert self.url_correct == True - - -class TestHTTPSIEMFriendly(TestHTTP): - modules_overrides = ["http"] - config_overrides = {"modules": {"http": dict(TestHTTP.config_overrides["modules"]["http"])}} - config_overrides["modules"]["http"]["siem_friendly"] = True - - def verify_data(self, j): - return j["data"] == {"DNS_NAME": "blacklanternsecurity.com"} and j["type"] == "DNS_NAME" diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index 27ed5a55e..364157421 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -23,13 +23,13 @@ def check(self, module_test, events): assert len(dns_json) == 1 dns_json = dns_json[0] scan = scan_json[0] - assert scan["data"]["name"] == module_test.scan.name - assert scan["data"]["id"] == module_test.scan.id + assert scan["data_json"]["name"] == module_test.scan.name + assert scan["data_json"]["id"] == module_test.scan.id assert scan["id"] == module_test.scan.id assert scan["uuid"] == str(module_test.scan.root_event.uuid) assert scan["parent_uuid"] == str(module_test.scan.root_event.uuid) - assert scan["data"]["target"]["seeds"] == ["blacklanternsecurity.com"] - assert scan["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"] + assert scan["data_json"]["target"]["seeds"] == ["blacklanternsecurity.com"] + assert scan["data_json"]["target"]["whitelist"] == ["blacklanternsecurity.com"] assert dns_json["data"] == dns_data assert dns_json["id"] == str(dns_event.id) assert dns_json["uuid"] == str(dns_event.uuid) @@ -53,18 +53,3 @@ def check(self, module_test, events): assert dns_reconstructed.discovery_context == context_data assert dns_reconstructed.discovery_path == [context_data] assert dns_reconstructed.parent_chain == [dns_json["uuid"]] - - -class TestJSONSIEMFriendly(ModuleTestBase): - modules_overrides = ["json"] - config_overrides = {"modules": {"json": {"siem_friendly": True}}} - - def check(self, module_test, events): - txt_file = module_test.scan.home / "output.json" - lines = list(module_test.scan.helpers.read_file(txt_file)) - passed = False - for line in lines: - e = json.loads(line) - if e["data"] == {"DNS_NAME": "blacklanternsecurity.com"}: - passed = True - assert passed diff --git a/bbot/test/test_step_2/module_tests/test_module_mongo.py b/bbot/test/test_step_2/module_tests/test_module_mongo.py new file mode 100644 index 000000000..ac28e64e7 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_mongo.py @@ -0,0 +1,151 @@ +import time +import asyncio + +from .base import ModuleTestBase + + +class TestMongo(ModuleTestBase): + test_db_name = "bbot_test" + test_collection_prefix = "test_" + config_overrides = { + "modules": { + "mongo": { + "database": test_db_name, + "username": "bbot", + "password": "bbotislife", + "collection_prefix": test_collection_prefix, + } + } + } + skip_distro_tests = True + + async def setup_before_prep(self, module_test): + + await asyncio.create_subprocess_exec( + "docker", + "run", + "--name", + "bbot-test-mongo", + "--rm", + "-e", + "MONGO_INITDB_ROOT_USERNAME=bbot", + "-e", + "MONGO_INITDB_ROOT_PASSWORD=bbotislife", + "-p", + "27017:27017", + "-d", + "mongo", + ) + + from motor.motor_asyncio import AsyncIOMotorClient + + # Connect to the MongoDB collection with retry logic + while True: + try: + client = AsyncIOMotorClient("mongodb://localhost:27017", username="bbot", password="bbotislife") + db = client[self.test_db_name] + events_collection = db.get_collection(self.test_collection_prefix + "events") + # Attempt a simple operation to confirm the connection + await events_collection.count_documents({}) + break # Exit the loop if connection is successful + except Exception as e: + print(f"Connection failed: {e}. Retrying...") + time.sleep(0.5) + + # Check that there are no events in the collection + count = await events_collection.count_documents({}) + assert count == 0, "There are existing events in the database" + + # Close the MongoDB connection + client.close() + + async def check(self, module_test, events): + try: + from bbot.models.pydantic import Event + from motor.motor_asyncio import AsyncIOMotorClient + + events_json = [e.json() for e in events] + events_json.sort(key=lambda x: x["timestamp"]) + + # Connect to the MongoDB collection + client = AsyncIOMotorClient("mongodb://localhost:27017", username="bbot", password="bbotislife") + db = client[self.test_db_name] + events_collection = db.get_collection(self.test_collection_prefix + "events") + + ### INDEXES ### + + # make sure the collection has all the right indexes + cursor = events_collection.list_indexes() + indexes = await cursor.to_list(length=None) + for field in Event._indexed_fields(): + assert any(field in index["key"] for index in indexes), f"Index for {field} not found" + + ### EVENTS ### + + # Fetch all events from the collection + cursor = events_collection.find({}) + db_events = await cursor.to_list(length=None) + + # make sure we have the same number of events + assert len(events_json) == len(db_events) + + for db_event in db_events: + assert isinstance(db_event["timestamp"], float) + assert isinstance(db_event["inserted_at"], float) + + # Convert to Pydantic objects and dump them + db_events_pydantic = [Event(**e).model_dump(exclude_none=True) for e in db_events] + db_events_pydantic.sort(key=lambda x: x["timestamp"]) + + # Find the main event with type DNS_NAME and data blacklanternsecurity.com + main_event = next( + ( + e + for e in db_events_pydantic + if e.get("type") == "DNS_NAME" and e.get("data") == "blacklanternsecurity.com" + ), + None, + ) + assert main_event is not None, "Main event with type DNS_NAME and data blacklanternsecurity.com not found" + + # Ensure it has the reverse_host attribute + expected_reverse_host = "blacklanternsecurity.com"[::-1] + assert ( + main_event.get("reverse_host") == expected_reverse_host + ), f"reverse_host attribute is not correct, expected {expected_reverse_host}" + + # Events don't match exactly because the mongo ones have reverse_host and inserted_at + assert events_json != db_events_pydantic + for db_event in db_events_pydantic: + db_event.pop("reverse_host") + db_event.pop("inserted_at") + # They should match after removing reverse_host + assert events_json == db_events_pydantic, "Events do not match" + + ### SCANS ### + + # Fetch all scans from the collection + cursor = db.get_collection(self.test_collection_prefix + "scans").find({}) + db_scans = await cursor.to_list(length=None) + assert len(db_scans) == 1, "There should be exactly one scan" + db_scan = db_scans[0] + assert db_scan["id"] == main_event["scan"], "Scan id should match main event scan" + + ### TARGETS ### + + # Fetch all targets from the collection + cursor = db.get_collection(self.test_collection_prefix + "targets").find({}) + db_targets = await cursor.to_list(length=None) + assert len(db_targets) == 1, "There should be exactly one target" + db_target = db_targets[0] + scan_event = next(e for e in events if e.type == "SCAN") + assert db_target["hash"] == scan_event.data["target"]["hash"], "Target hash should match scan target hash" + + finally: + # Clean up: Delete all documents in the collection + await events_collection.delete_many({}) + # Close the MongoDB connection + client.close() + await asyncio.create_subprocess_exec( + "docker", "stop", "bbot-test-mongo", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) diff --git a/bbot/test/test_step_2/module_tests/test_module_splunk.py b/bbot/test/test_step_2/module_tests/test_module_splunk.py index d55ed17c2..eef148944 100644 --- a/bbot/test/test_step_2/module_tests/test_module_splunk.py +++ b/bbot/test/test_step_2/module_tests/test_module_splunk.py @@ -23,7 +23,7 @@ def verify_data(self, j): if not j["index"] == "bbot_index": return False data = j["event"] - if not data["data"] == "blacklanternsecurity.com" and data["type"] == "DNS_NAME": + if not data["data_json"] == "blacklanternsecurity.com" and data["type"] == "DNS_NAME": return False return True diff --git a/bbot/test/test_step_2/module_tests/test_module_sqlite.py b/bbot/test/test_step_2/module_tests/test_module_sqlite.py index ec80b7555..7970627b1 100644 --- a/bbot/test/test_step_2/module_tests/test_module_sqlite.py +++ b/bbot/test/test_step_2/module_tests/test_module_sqlite.py @@ -8,6 +8,8 @@ class TestSQLite(ModuleTestBase): def check(self, module_test, events): sqlite_output_file = module_test.scan.home / "output.sqlite" assert sqlite_output_file.exists(), "SQLite output file not found" + + # first connect with raw sqlite with sqlite3.connect(sqlite_output_file) as db: cursor = db.cursor() results = cursor.execute("SELECT * FROM event").fetchall() @@ -16,3 +18,15 @@ def check(self, module_test, events): assert len(results) == 1, "No scans found in SQLite database" results = cursor.execute("SELECT * FROM target").fetchall() assert len(results) == 1, "No targets found in SQLite database" + + # then connect with bbot models + from bbot.models.sql import Event + from sqlmodel import create_engine, Session, select + + engine = create_engine(f"sqlite:///{sqlite_output_file}") + + with Session(engine) as session: + statement = select(Event).where(Event.host == "evilcorp.com") + event = session.exec(statement).first() + assert event.host == "evilcorp.com", "Event host should match target host" + assert event.data == "evilcorp.com", "Event data should match target host" diff --git a/docs/scanning/output.md b/docs/scanning/output.md index dd45a5c83..16cfbd359 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -155,15 +155,20 @@ config: ### Elasticsearch -When outputting to Elastic, use the `http` output module with the following settings (replace `` with your desired index, e.g. `bbot`): +- Step 1: Spin up a quick Elasticsearch docker image + +```bash +docker run -d -p 9200:9200 --name=bbot-elastic --v "$(pwd)/elastic_data:/usr/share/elasticsearch/data" -e ELASTIC_PASSWORD=bbotislife -m 1GB docker.elastic.co/elasticsearch/elasticsearch:8.16.0 +``` + +- Step 2: Execute a scan with `elastic` output module ```bash # send scan results directly to elasticsearch -bbot -t evilcorp.com -om http -c \ - modules.http.url=http://localhost:8000//_doc \ - modules.http.siem_friendly=true \ - modules.http.username=elastic \ - modules.http.password=changeme +# note: you can replace "bbot_events" with your own index name +bbot -t evilcorp.com -om elastic -c \ + modules.elastic.url=https://localhost:9200/bbot_events/_doc \ + modules.elastic.password=bbotislife ``` Alternatively, via a preset: @@ -171,11 +176,9 @@ Alternatively, via a preset: ```yaml title="elastic_preset.yml" config: modules: - http: - url: http://localhost:8000//_doc - siem_friendly: true - username: elastic - password: changeme + elastic: + url: http://localhost:9200/bbot_events/_doc + password: bbotislife ``` ### Splunk diff --git a/docs/scanning/tips_and_tricks.md b/docs/scanning/tips_and_tricks.md index c5073c1d6..e13d82875 100644 --- a/docs/scanning/tips_and_tricks.md +++ b/docs/scanning/tips_and_tricks.md @@ -108,24 +108,6 @@ config: bbot -t evilcorp.com -p skip_cdns.yml ``` -### Ingest BBOT Data Into SIEM (Elastic, Splunk) - -If your goal is to run a BBOT scan and later feed its data into a SIEM such as Elastic, be sure to enable this option when scanning: - -```bash -bbot -t evilcorp.com -c modules.json.siem_friendly=true -``` - -This ensures the `.data` event attribute is always the same type (a dictionary), by nesting it like so: -```json -{ - "type": "DNS_NAME", - "data": { - "DNS_NAME": "blacklanternsecurity.com" - } -} -``` - ### Custom HTTP Proxy Web pentesters may appreciate BBOT's ability to quickly populate Burp Suite site maps for all subdomains in a target. If your scan includes gowitness, this will capture the traffic as if you manually visited each website in your browser -- including auxiliary web resources and javascript API calls. To accomplish this, set the `web.http_proxy` config option like so: