From f7bdabaef477dcc1de9e3445a38ec4d91a0dabf4 Mon Sep 17 00:00:00 2001 From: w0Tx <98209373+w0Tx@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:43:39 +0100 Subject: [PATCH 1/6] Add splunk.py output module Add support to send JSON data to splunk instance via HTTP Event Collector --- bbot/modules/output/splunk.py | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 bbot/modules/output/splunk.py diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py new file mode 100644 index 000000000..704aff542 --- /dev/null +++ b/bbot/modules/output/splunk.py @@ -0,0 +1,59 @@ +from bbot.core.errors import RequestError + +from bbot.modules.output.base import BaseOutputModule + +class SPLUNK(BaseOutputModule): + watched_events = ["*"] + meta = {"description": "Send every event to a splunk instance through HTTP Event Collector"} + options = { + "url": "", + "hectoken": "", + "index": "", + "source": "", + "timeout": 10, + } + options_desc = { + "url": "Web URL", + "hectoken": "HEC Token", + "index": "Index to send data to", + "source": "Source path to be added to the metadata", + "timeout": "HTTP timeout", + } + + async def setup(self): + self.url = self.config.get("url", "") + self.source = self.config.get("source", "") + self.index = self.config.get("index", "main") + self.timeout = self.config.get("timeout", 10) + self.headers = {} + + hectoken = self.config.get("hectoken", "") + if hectoken: + self.headers["Authorization"] = f"Splunk {hectoken}" + if not self.url: + self.warning("Must set URL") + return False + if not self.source: + self.warning("Please provide a source") + return True + + async def handle_event(self, event): + while 1: + try: + data = { + "index": self.index, + "source": self.source, + "sourcetype": "_json", + "event": dict(event), + } + await self.helpers.request( + url=self.url, + method="POST", + headers=self.headers, + json=data, + raise_error=True, + ) + break + except RequestError as e: + self.warning(f"Error sending {event}: {e}, retrying...") + await self.helpers.sleep(1) From 5f7596e9d897b16ec90e6bbac13ab290dee0e310 Mon Sep 17 00:00:00 2001 From: w0Tx <98209373+w0Tx@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:45:01 +0100 Subject: [PATCH 2/6] Add doc for splunk output module Add documentation for splunk output module --- docs/scanning/output.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/scanning/output.md b/docs/scanning/output.md index 81b4b8ede..16c317336 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -135,6 +135,25 @@ output_modules: password: P@ssw0rd ``` +### Splunk + +The `splunk` output module sends [events](events.md) in JSON format to a desired splunk instance via [HEC](https://docs.splunk.com/Documentation/Splunk/9.2.0/Data/UsetheHTTPEventCollector) + +You can customize this output with the following variables : + +```yaml title="~/.bbot/config/bbot.yml" +output_modules: + splunk: + #The full URL with the URI `/services/collector/event` + url: https://localhost:8088/services/collector/event + #Has to be generated from splunk webui + hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + #Defaults to `main` if not set + index: my-specific-index + #Defaults to `bbot` if not set + source: /my/source.json +``` + ### Asset Inventory The `asset_inventory` module produces a CSV like this: From a4f030d949efcaa57fd820b8a9af1af4d8d6a6b3 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 17 Feb 2024 17:56:46 -0500 Subject: [PATCH 3/6] splunk tests --- bbot/modules/output/splunk.py | 8 +-- .../module_tests/test_module_splunk.py | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 bbot/test/test_step_2/module_tests/test_module_splunk.py diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py index 704aff542..25ddfef3d 100644 --- a/bbot/modules/output/splunk.py +++ b/bbot/modules/output/splunk.py @@ -2,7 +2,8 @@ from bbot.modules.output.base import BaseOutputModule -class SPLUNK(BaseOutputModule): + +class Splunk(BaseOutputModule): watched_events = ["*"] meta = {"description": "Send every event to a splunk instance through HTTP Event Collector"} options = { @@ -31,8 +32,7 @@ async def setup(self): if hectoken: self.headers["Authorization"] = f"Splunk {hectoken}" if not self.url: - self.warning("Must set URL") - return False + return False, "Must set URL" if not self.source: self.warning("Please provide a source") return True @@ -44,7 +44,7 @@ async def handle_event(self, event): "index": self.index, "source": self.source, "sourcetype": "_json", - "event": dict(event), + "event": event.json(), } await self.helpers.request( url=self.url, diff --git a/bbot/test/test_step_2/module_tests/test_module_splunk.py b/bbot/test/test_step_2/module_tests/test_module_splunk.py new file mode 100644 index 000000000..67d67a4ef --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_splunk.py @@ -0,0 +1,58 @@ +import json +import httpx + +from .base import ModuleTestBase + + +class TestSplunk(ModuleTestBase): + downstream_url = "https://splunk.blacklanternsecurity.fakedomain:1234/services/collector" + config_overrides = { + "output_modules": { + "splunk": { + "url": downstream_url, + "hectoken": "HECTOKEN", + "index": "bbot_index", + "source": "bbot_source", + } + } + } + + def verify_data(self, j): + if not j["source"] == "bbot_source": + return False + if not j["index"] == "bbot_index": + return False + data = j["event"] + if not data["data"] == "blacklanternsecurity.com" and data["type"] == "DNS_NAME": + return False + return True + + async def setup_after_prep(self, module_test): + self.url_correct = False + self.method_correct = False + self.got_event = False + self.headers_correct = False + + async def custom_callback(request): + j = json.loads(request.content) + if request.url == self.downstream_url: + self.url_correct = True + if request.method == "POST": + self.method_correct = True + if "Authorization" in request.headers: + self.headers_correct = True + if self.verify_data(j): + self.got_event = True + return httpx.Response( + status_code=200, + ) + + module_test.httpx_mock.add_callback(custom_callback) + module_test.httpx_mock.add_callback(custom_callback) + module_test.httpx_mock.add_response() + + def check(self, module_test, events): + assert self.got_event == True + assert self.headers_correct == True + assert self.method_correct == True + assert self.url_correct == True From 4b94b75db0a8f88fdd073646ab4d2cf20697093f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 17 Feb 2024 18:01:44 -0500 Subject: [PATCH 4/6] updated docs --- bbot/modules/output/splunk.py | 2 +- docs/scanning/output.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py index 25ddfef3d..242f1759e 100644 --- a/bbot/modules/output/splunk.py +++ b/bbot/modules/output/splunk.py @@ -23,7 +23,7 @@ class Splunk(BaseOutputModule): async def setup(self): self.url = self.config.get("url", "") - self.source = self.config.get("source", "") + self.source = self.config.get("source", "bbot") self.index = self.config.get("index", "main") self.timeout = self.config.get("timeout", 10) self.headers = {} diff --git a/docs/scanning/output.md b/docs/scanning/output.md index 16c317336..376585ebe 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -137,20 +137,20 @@ output_modules: ### Splunk -The `splunk` output module sends [events](events.md) in JSON format to a desired splunk instance via [HEC](https://docs.splunk.com/Documentation/Splunk/9.2.0/Data/UsetheHTTPEventCollector) +The `splunk` output module sends [events](events.md) in JSON format to a desired splunk instance via [HEC](https://docs.splunk.com/Documentation/Splunk/9.2.0/Data/UsetheHTTPEventCollector). -You can customize this output with the following variables : +You can customize this output with the following config options: ```yaml title="~/.bbot/config/bbot.yml" output_modules: splunk: - #The full URL with the URI `/services/collector/event` + # The full URL with the URI `/services/collector/event` url: https://localhost:8088/services/collector/event - #Has to be generated from splunk webui + # Generated from splunk webui hectoken: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - #Defaults to `main` if not set + # Defaults to `main` if not set index: my-specific-index - #Defaults to `bbot` if not set + # Defaults to `bbot` if not set source: /my/source.json ``` From 2e705228f0768fa1828c173a08673782fe3f25b6 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 17 Feb 2024 20:38:39 -0500 Subject: [PATCH 5/6] add output module list in README --- README.md | 24 +++++++++++++++++++++++- bbot/scripts/docs.py | 5 +++++ docs/modules/list_of_modules.md | 1 + docs/scanning/advanced.md | 16 +++++----------- docs/scanning/configuration.md | 5 +++++ docs/scanning/events.md | 2 +- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6b0cca3be..7d19fa630 100644 --- a/README.md +++ b/README.md @@ -280,4 +280,26 @@ For a full list of modules, including the data types consumed and emitted by eac | subdomain-hijack | 1 | Detects hijackable subdomains | subdomain_hijack | | web-screenshots | 1 | Takes screenshots of web pages | gowitness | - + +## BBOT Output Modules +BBOT can save its data to TXT, CSV, JSON, and tons of other destinations including [Neo4j](https://www.blacklanternsecurity.com/bbot/scanning/output/#neo4j), [Splunk](https://www.blacklanternsecurity.com/bbot/scanning/output/#splunk), and [Discord](https://www.blacklanternsecurity.com/bbot/scanning/output/#discord-slack-teams). For instructions on how to use these, see [Output Modules](https://www.blacklanternsecurity.com/bbot/scanning/output). + + +| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | +|-----------------|--------|-----------------|-----------------------------------------------------------------------------------------|----------------|--------------------------------------------------------------------------------------------------|---------------------------| +| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | +| csv | output | No | Output to CSV | | * | | +| discord | output | No | Message a Discord channel when certain events are encountered | | * | | +| emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | +| http | output | No | Send every event to a custom URL via a web request | | * | | +| human | output | No | Output to text | | * | | +| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | +| neo4j | output | No | Output to Neo4j | | * | | +| python | output | No | Output via Python API | | * | | +| slack | output | No | Message a Slack channel when certain events are encountered | | * | | +| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | +| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | +| teams | output | No | Message a Teams channel when certain events are encountered | | * | | +| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | +| websocket | output | No | Output to websockets | | * | | + diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index dcf9cd710..8e6d045f3 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -94,6 +94,11 @@ def update_individual_module_options(): assert len(bbot_module_table.splitlines()) > 50 update_md_files("BBOT MODULES", bbot_module_table) + # BBOT output modules + bbot_output_module_table = module_loader.modules_table(mod_type="output") + assert len(bbot_output_module_table.splitlines()) > 10 + update_md_files("BBOT OUTPUT MODULES", bbot_output_module_table) + # BBOT module options bbot_module_options_table = module_loader.modules_options_table() assert len(bbot_module_options_table.splitlines()) > 100 diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index a3ffc76c6..ebf4f182f 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -107,6 +107,7 @@ | neo4j | output | No | Output to Neo4j | | * | | | python | output | No | Output via Python API | | * | | | slack | output | No | Message a Slack channel when certain events are encountered | | * | | +| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | | subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | | teams | output | No | Message a Teams channel when certain events are encountered | | * | | | web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 0baaf35c8..8207b7ce7 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -33,16 +33,10 @@ asyncio.run(main()) ```text -usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] - [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] - [--strict-scope] [-m MODULE [MODULE ...]] [-l] - [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] - [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] - [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] - [-o DIR] [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] - [--dry-run] [--current-config] - [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] - [-a] [--version] +usage: bbot [-h] [--help-all] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-m MODULE [MODULE ...]] [-l] + [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [-om MODULE [MODULE ...]] [--allow-deadly] [-n SCAN_NAME] [-o DIR] + [-c [CONFIG ...]] [-v] [-d] [-s] [--force] [-y] [--dry-run] [--current-config] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [-a] + [--version] Bighuge BLS OSINT Tool @@ -73,7 +67,7 @@ Modules: -ef FLAG [FLAG ...], --exclude-flags FLAG [FLAG ...] Disable modules with these flags. (e.g. -ef aggressive) -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] - Output module(s). Choices: asset_inventory,csv,discord,emails,http,human,json,neo4j,python,slack,subdomains,teams,web_report,websocket + Output module(s). Choices: asset_inventory,csv,discord,emails,http,human,json,neo4j,python,slack,splunk,subdomains,teams,web_report,websocket --allow-deadly Enable the use of highly aggressive modules Scan: diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 21db76264..babe891b1 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -364,6 +364,11 @@ Many modules accept their own configuration options. These options have the abil | output_modules.slack.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | | output_modules.slack.min_severity | str | Only allow VULNERABILITY events of this severity or higher | LOW | | output_modules.slack.webhook_url | str | Discord webhook URL | | +| output_modules.splunk.hectoken | str | HEC Token | | +| output_modules.splunk.index | str | Index to send data to | | +| output_modules.splunk.source | str | Source path to be added to the metadata | | +| output_modules.splunk.timeout | int | HTTP timeout | 10 | +| output_modules.splunk.url | str | Web URL | | | output_modules.subdomains.include_unresolved | bool | Include unresolved subdomains in output | False | | output_modules.subdomains.output_file | str | Output to file | | | output_modules.teams.event_types | list | Types of events to send | ['VULNERABILITY', 'FINDING'] | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 6628fac46..d2aaa4595 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -51,7 +51,7 @@ Below is a full list of event types along with which modules produce/consume the | Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | |---------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| * | 11 | 0 | affiliates, csv, discord, http, human, json, neo4j, python, slack, teams, websocket | | +| * | 12 | 0 | affiliates, csv, discord, http, human, json, neo4j, python, slack, splunk, teams, websocket | | | ASN | 0 | 1 | | asn | | AZURE_TENANT | 1 | 0 | speculate | | | CODE_REPOSITORY | 0 | 2 | | github_codesearch, github_org | From 6967bce5b6d68ee549ff440756058fdb5f930d1c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Sat, 17 Feb 2024 20:43:09 -0500 Subject: [PATCH 6/6] update output module docs --- docs/scanning/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scanning/output.md b/docs/scanning/output.md index 376585ebe..af1db4737 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -1,6 +1,6 @@ # Output -By default, BBOT saves its output in TXT, JSON, and CSV formats: +By default, BBOT saves its output in TXT, JSON, and CSV formats. The filenames are logged at the end of each scan: ![bbot output](https://github.com/blacklanternsecurity/bbot/assets/20261699/bb3da441-2682-408f-b955-19b268823b82) Every BBOT scan gets a unique and mildly-entertaining name like **`demonic_jimmy`**. Output for that scan, including scan stats and any web screenshots, etc., are saved to a folder by that name in `~/.bbot/scans`. The most recent 20 scans are kept, and older ones are removed. You can change the location of BBOT's output with `--output`, and you can also pick a custom scan name with `--name`.