Skip to content

Commit

Permalink
Merge pull request #1234 from blacklanternsecurity/stdout-module
Browse files Browse the repository at this point in the history
Stdout Output Module
  • Loading branch information
TheTechromancer authored Apr 26, 2024
2 parents afe9a35 + e92e961 commit 3942b65
Show file tree
Hide file tree
Showing 29 changed files with 835 additions and 513 deletions.
31 changes: 18 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -215,23 +215,28 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/scann
- [List of Modules](https://www.blacklanternsecurity.com/bbot/modules/list_of_modules)
- [Nuclei](https://www.blacklanternsecurity.com/bbot/modules/nuclei)
- **Misc**
- [Contribution](https://www.blacklanternsecurity.com/bbot/contribution)
- [Release History](https://www.blacklanternsecurity.com/bbot/release_history)
- [Troubleshooting](https://www.blacklanternsecurity.com/bbot/troubleshooting)
- **Developer Manual**
- [How to Write a Module](https://www.blacklanternsecurity.com/bbot/contribution)
- [Development Overview](https://www.blacklanternsecurity.com/bbot/dev/)
- [Scanner](https://www.blacklanternsecurity.com/bbot/dev/scanner)
- [Event](https://www.blacklanternsecurity.com/bbot/dev/event)
- [Target](https://www.blacklanternsecurity.com/bbot/dev/target)
- [BaseModule](https://www.blacklanternsecurity.com/bbot/dev/basemodule)
- **Helpers**
- [Overview](https://www.blacklanternsecurity.com/bbot/dev/helpers/)
- [Command](https://www.blacklanternsecurity.com/bbot/dev/helpers/command)
- [DNS](https://www.blacklanternsecurity.com/bbot/dev/helpers/dns)
- [Interactsh](https://www.blacklanternsecurity.com/bbot/dev/helpers/interactsh)
- [Miscellaneous](https://www.blacklanternsecurity.com/bbot/dev/helpers/misc)
- [Web](https://www.blacklanternsecurity.com/bbot/dev/helpers/web)
- [Word Cloud](https://www.blacklanternsecurity.com/bbot/dev/helpers/wordcloud)
- [How to Write a BBOT Module](https://www.blacklanternsecurity.com/bbot/dev/module_howto)
- [Discord Bot Example](https://www.blacklanternsecurity.com/bbot/dev/discord_bot)
- **Code Reference**
- [Scanner](https://www.blacklanternsecurity.com/bbot/dev/scanner)
- [Presets](https://www.blacklanternsecurity.com/bbot/dev/presets)
- [Event](https://www.blacklanternsecurity.com/bbot/dev/event)
- [Target](https://www.blacklanternsecurity.com/bbot/dev/target)
- [BaseModule](https://www.blacklanternsecurity.com/bbot/dev/basemodule)
- [BBOTCore](https://www.blacklanternsecurity.com/bbot/dev/core)
- **Helpers**
- [Overview](https://www.blacklanternsecurity.com/bbot/dev/helpers/)
- [Command](https://www.blacklanternsecurity.com/bbot/dev/helpers/command)
- [DNS](https://www.blacklanternsecurity.com/bbot/dev/helpers/dns)
- [Interactsh](https://www.blacklanternsecurity.com/bbot/dev/helpers/interactsh)
- [Miscellaneous](https://www.blacklanternsecurity.com/bbot/dev/helpers/misc)
- [Web](https://www.blacklanternsecurity.com/bbot/dev/helpers/web)
- [Word Cloud](https://www.blacklanternsecurity.com/bbot/dev/helpers/wordcloud)
<!-- END BBOT DOCS TOC -->

## Contribution
Expand Down
44 changes: 26 additions & 18 deletions bbot/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,23 +58,23 @@ async def _main():

# print help if no arguments
if len(sys.argv) == 1:
log.stdout(preset.args.parser.format_help())
print(preset.args.parser.format_help())
sys.exit(1)
return

# --version
if options.version:
log.stdout(__version__)
print(__version__)
sys.exit(0)
return

# --list-presets
if options.list_presets:
log.stdout("")
log.stdout("### PRESETS ###")
log.stdout("")
print("")
print("### PRESETS ###")
print("")
for row in preset.presets_table().splitlines():
log.stdout(row)
print(row)
return

# if we're listing modules or their options
Expand All @@ -86,34 +86,38 @@ async def _main():
module_type = preloaded.get("type", "scan")
preset.add_module(module, module_type=module_type)

if options.modules or options.output_modules or options.flags:
preset._default_output_modules = options.output_modules
preset._default_internal_modules = []

preset.bake()

# --list-modules
if options.list_modules:
log.stdout("")
log.stdout("### MODULES ###")
log.stdout("")
print("")
print("### MODULES ###")
print("")
for row in preset.module_loader.modules_table(preset.modules).splitlines():
log.stdout(row)
print(row)
return

# --list-module-options
if options.list_module_options:
log.stdout("")
log.stdout("### MODULE OPTIONS ###")
log.stdout("")
print("")
print("### MODULE OPTIONS ###")
print("")
for row in preset.module_loader.modules_options_table(preset.modules).splitlines():
log.stdout(row)
print(row)
return

# --list-flags
if options.list_flags:
flags = preset.flags if preset.flags else None
log.stdout("")
log.stdout("### FLAGS ###")
log.stdout("")
print("")
print("### FLAGS ###")
print("")
for row in preset.module_loader.flags_table(flags=flags).splitlines():
log.stdout(row)
print(row)
return

try:
Expand Down Expand Up @@ -228,6 +232,10 @@ async def akeyboard_listen():

return True

except BBOTError as e:
log.error(str(e))
log.trace(traceback.format_exc())

finally:
# save word cloud
with suppress(BaseException):
Expand Down
20 changes: 4 additions & 16 deletions bbot/core/config/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ class BBOTLogger:

def __init__(self, core):
# custom logging levels
if getattr(logging, "STDOUT", None) is None:
self.addLoggingLevel("STDOUT", 100)
if getattr(logging, "HUGEWARNING", None) is None:
self.addLoggingLevel("TRACE", 49)
self.addLoggingLevel("HUGEWARNING", 31)
self.addLoggingLevel("HUGESUCCESS", 26)
Expand Down Expand Up @@ -191,9 +190,7 @@ def log_handlers(self):
)

def stderr_filter(record):
if record.levelno == logging.STDOUT or (
record.levelno == logging.TRACE and self.log_level > logging.DEBUG
):
if record.levelno == logging.TRACE and self.log_level > logging.DEBUG:
return False
if record.levelno < self.log_level:
return False
Expand All @@ -202,26 +199,17 @@ def stderr_filter(record):
# Log to stderr
stderr_handler = logging.StreamHandler(sys.stderr)
stderr_handler.addFilter(stderr_filter)
# Log to stdout
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.addFilter(lambda x: x.levelno == logging.STDOUT)
# log to files
debug_handler.addFilter(
lambda x: x.levelno == logging.TRACE or (x.levelno < logging.VERBOSE and x.levelno != logging.STDOUT)
)
main_handler.addFilter(
lambda x: x.levelno not in (logging.STDOUT, logging.TRACE) and x.levelno >= logging.VERBOSE
)
debug_handler.addFilter(lambda x: x.levelno == logging.TRACE or (x.levelno < logging.VERBOSE))
main_handler.addFilter(lambda x: x.levelno != logging.TRACE and x.levelno >= logging.VERBOSE)

# Set log format
debug_handler.setFormatter(debug_format)
main_handler.setFormatter(debug_format)
stderr_handler.setFormatter(ColoredFormatter("%(levelname)s %(name)s: %(message)s"))
stdout_handler.setFormatter(logging.Formatter("%(message)s"))

self._log_handlers = {
"stderr": stderr_handler,
"stdout": stdout_handler,
"file_debug": debug_handler,
"file_main": main_handler,
}
Expand Down
14 changes: 0 additions & 14 deletions bbot/modules/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1194,20 +1194,6 @@ def preserve_graph(self):
preserve_graph = self._preserve_graph
return preserve_graph

def stdout(self, *args, **kwargs):
"""Writes log messages directly to standard output.
This is typically reserved for output modules only, e.g. `human` or `json`.
Args:
*args: Variable length argument list to be passed to `self.log.stdout`.
**kwargs: Arbitrary keyword arguments to be passed to `self.log.stdout`.
Examples:
>>> self.stdout("This will be printed to stdout")
"""
self.log.stdout(*args, extra={"scan_id": self.scan.id}, **kwargs)

def debug(self, *args, trace=False, **kwargs):
"""Logs debug messages and optionally the stack trace of the most recent exception.
Expand Down
8 changes: 8 additions & 0 deletions bbot/modules/output/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ class BaseOutputModule(BaseModule):
scope_distance_modifier = None
_stats_exclude = True

def human_event_str(self, event):
event_type = f"[{event.type}]"
event_tags = ""
if getattr(event, "tags", []):
event_tags = f'\t({", ".join(sorted(getattr(event, "tags", [])))})'
event_str = f"{event_type:<20}\t{event.data_human}\t{event.module_sequence}{event_tags}"
return event_str

def _event_precheck(self, event):
# special signal event types
if event.type in ("FINISHED",):
Expand Down
5 changes: 3 additions & 2 deletions bbot/modules/output/emails.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
from bbot.modules.output.txt import TXT
from bbot.modules.base import BaseModule
from bbot.modules.output.human import Human


class Emails(Human):
class Emails(TXT):
watched_events = ["EMAIL_ADDRESS"]
meta = {"description": "Output any email addresses found belonging to the target domain"}
options = {"output_file": ""}
options_desc = {"output_file": "Output to file"}
in_scope_only = True
accept_dupes = False

output_filename = "emails.txt"

Expand Down
49 changes: 0 additions & 49 deletions bbot/modules/output/human.py

This file was deleted.

5 changes: 1 addition & 4 deletions bbot/modules/output/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,9 @@
class JSON(BaseOutputModule):
watched_events = ["*"]
meta = {"description": "Output to Newline-Delimited JSON (NDJSON)"}
options = {"output_file": "", "console": False, "siem_friendly": False}
options = {"output_file": "", "siem_friendly": False}
options_desc = {
"output_file": "Output to file",
"console": "Output to console",
"siem_friendly": "Output JSON in a SIEM-friendly format for ingestion into Elastic, Splunk, etc.",
}
_preserve_graph = True
Expand All @@ -26,8 +25,6 @@ async def handle_event(self, event):
if self.file is not None:
self.file.write(event_str + "\n")
self.file.flush()
if self.config.get("console", False) or "human" not in self.scan.modules:
self.stdout(event_str)

async def cleanup(self):
if getattr(self, "_file", None) is not None:
Expand Down
68 changes: 68 additions & 0 deletions bbot/modules/output/stdout.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import json

from bbot.logger import log_to_stderr
from bbot.modules.output.base import BaseOutputModule


class Stdout(BaseOutputModule):
watched_events = ["*"]
meta = {"description": "Output to text"}
options = {"format": "text", "event_types": [], "event_fields": [], "in_scope_only": False, "accept_dupes": True}
options_desc = {
"format": "Which text format to display, choices: text,json",
"event_types": "Which events to display, default all event types",
"event_fields": "Which event fields to display",
"in_scope_only": "Whether to only show in-scope events",
"accept_dupes": "Whether to show duplicate events, default True",
}
vuln_severity_map = {"LOW": "HUGEWARNING", "MEDIUM": "HUGEWARNING", "HIGH": "CRITICAL", "CRITICAL": "CRITICAL"}
format_choices = ["text", "json"]

async def setup(self):
self.text_format = self.config.get("format", "text").strip().lower()
if not self.text_format in self.format_choices:
return (
False,
f'Invalid text format choice, "{self.text_format}" (choices: {",".join(self.format_choices)})',
)
self.accept_event_types = [str(s).upper() for s in self.config.get("event_types", [])]
self.show_event_fields = [str(s) for s in self.config.get("event_fields", [])]
self.in_scope_only = self.config.get("in_scope_only", False)
self.accept_dupes = self.config.get("accept_dupes", False)
return True

async def filter_event(self, event):
if self.accept_event_types:
if not event.type in self.accept_event_types:
return False, f'Event type "{event.type}" is not in the allowed event_types'
return True

async def handle_event(self, event):
event_json = event.json(mode="human")
if self.show_event_fields:
event_json = {k: str(event_json.get(k, "")) for k in self.show_event_fields}

if self.text_format == "text":
await self.handle_text(event, event_json)
elif self.text_format == "json":
await self.handle_json(event, event_json)

async def handle_text(self, event, event_json):
if self.show_event_fields:
event_str = "\t".join([str(s) for s in event_json.values()])
else:
event_str = self.human_event_str(event)

# log vulnerabilities in vivid colors
if event.type == "VULNERABILITY":
severity = event.data.get("severity", "INFO")
if severity in self.vuln_severity_map:
loglevel = self.vuln_severity_map[severity]
log_to_stderr(event_str, level=loglevel, logname=False)
elif event.type == "FINDING":
log_to_stderr(event_str, level="HUGEINFO", logname=False)

print(event_str)

async def handle_json(self, event, event_json):
print(json.dumps(event_json))
4 changes: 2 additions & 2 deletions bbot/modules/output/subdomains.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from bbot.modules.output.txt import TXT
from bbot.modules.base import BaseModule
from bbot.modules.output.human import Human


class Subdomains(Human):
class Subdomains(TXT):
watched_events = ["DNS_NAME", "DNS_NAME_UNRESOLVED"]
meta = {"description": "Output only resolved, in-scope subdomains"}
options = {"output_file": "", "include_unresolved": False}
Expand Down
Loading

0 comments on commit 3942b65

Please sign in to comment.