From 995358f2718f34e553fba38e4b1092463cee4d1e Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 16 Sep 2023 19:27:46 -0500 Subject: [PATCH 1/7] Update json.py to NDJSON terminology Change output.json to output.ndjson, and update the output text to the end user for clarification. --- bbot/modules/output/json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index 561354c45..5b1c95195 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -6,12 +6,12 @@ class JSON(BaseOutputModule): watched_events = ["*"] - meta = {"description": "Output to JSON"} + meta = {"description": "Output to NDJSON"} options = {"output_file": "", "console": False} options_desc = {"output_file": "Output to file", "console": "Output to console"} async def setup(self): - self._prep_output_dir("output.json") + self._prep_output_dir("output.ndjson") return True async def handle_event(self, event): @@ -29,4 +29,4 @@ async def cleanup(self): async def report(self): if self._file is not None: - self.info(f"Saved JSON output to {self.output_file}") + self.info(f"Saved NDJSON output to {self.output_file}") From 82b6f91b39a98b4d7b6096e8e358ed675676fa70 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 16 Sep 2023 19:32:45 -0500 Subject: [PATCH 2/7] Update output.md to use output.ndjson Update docs to reflect .ndjson file change --- docs/scanning/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/scanning/output.md b/docs/scanning/output.md index edfb0c992..b2ae1c866 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -64,7 +64,7 @@ You can filter on the JSON output with `jq`: ```bash # pull out only the .data attribute of every DNS_NAME -$ jq -r 'select(.type=="DNS_NAME") | .data' ~/.bbot/scans/extreme_johnny/output.json +$ jq -r 'select(.type=="DNS_NAME") | .data' ~/.bbot/scans/extreme_johnny/output.ndjson evilcorp.com www.evilcorp.com mail.evilcorp.com From 3170cadbe108ab7f7ffd8879c8a103c9a35faf90 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 16 Sep 2023 19:34:06 -0500 Subject: [PATCH 3/7] Rename test_output.json to test_output.ndjson Updating to .ndjson --- bbot/test/{test_output.json => test_output.ndjson} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bbot/test/{test_output.json => test_output.ndjson} (100%) diff --git a/bbot/test/test_output.json b/bbot/test/test_output.ndjson similarity index 100% rename from bbot/test/test_output.json rename to bbot/test/test_output.ndjson From 4935d03324d724ba7c8afc906d6b5b54e6753e96 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 16 Sep 2023 19:36:00 -0500 Subject: [PATCH 4/7] Update test_python_api.py to use output.ndjson Updating for output.ndjson change --- bbot/test/test_step_1/test_python_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 7a632ed3d..b393158b6 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -14,7 +14,7 @@ async def test_python_api(bbot_config): # make sure output files work scan2 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="python_api_test") await scan2.async_start_without_generator() - out_file = scan2.helpers.scans_dir / "python_api_test" / "output.json" + out_file = scan2.helpers.scans_dir / "python_api_test" / "output.ndjson" assert list(scan2.helpers.read_file(out_file)) # make sure config loads properly bbot_home = "/tmp/.bbot_python_api_test" @@ -34,7 +34,7 @@ def test_python_api_sync(bbot_config): # make sure output files work scan2 = Scanner("127.0.0.1", config=bbot_config, output_modules=["json"], name="python_api_test") scan2.start_without_generator() - out_file = scan2.helpers.scans_dir / "python_api_test" / "output.json" + out_file = scan2.helpers.scans_dir / "python_api_test" / "output.ndjson" assert list(scan2.helpers.read_file(out_file)) # make sure config loads properly bbot_home = "/tmp/.bbot_python_api_test" From d3c84e4fb4be9affad1f2a9313190873cbe9c153 Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 16 Sep 2023 19:37:12 -0500 Subject: [PATCH 5/7] Update test_module_json.py to use output.ndjson Updating for ndjson filename use. --- bbot/test/test_step_2/module_tests/test_module_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_json.py b/bbot/test/test_step_2/module_tests/test_module_json.py index dd552742a..6dafb68a5 100644 --- a/bbot/test/test_step_2/module_tests/test_module_json.py +++ b/bbot/test/test_step_2/module_tests/test_module_json.py @@ -6,7 +6,7 @@ class TestJSON(ModuleTestBase): def check(self, module_test, events): - txt_file = module_test.scan.home / "output.json" + txt_file = module_test.scan.home / "output.ndjson" lines = list(module_test.scan.helpers.read_file(txt_file)) assert lines e = event_from_json(json.loads(lines[0])) From d7b5673d35597d7807fcfc96586bdb22502bbe7f Mon Sep 17 00:00:00 2001 From: Nic Date: Sat, 16 Sep 2023 19:39:02 -0500 Subject: [PATCH 6/7] Update test_cli.py to use output.ndjson Update filename to make use of ndjson extension. --- bbot/test/test_step_1/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index a0e821a12..e2b7a2583 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -42,7 +42,7 @@ async def test_cli(monkeypatch, bbot_config): assert (scan_home / "wordcloud.tsv").is_file() assert (scan_home / "output.txt").is_file() assert (scan_home / "output.csv").is_file() - assert (scan_home / "output.json").is_file() + assert (scan_home / "output.ndjson").is_file() with open(scan_home / "output.csv") as f: lines = f.readlines() assert lines[0] == "Event type,Event data,IP Address,Source Module,Scope Distance,Event Tags\n" From 9c6c490edf3b630825c2a5c9e12defb302d31b25 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 18 Sep 2023 08:58:02 -0400 Subject: [PATCH 7/7] fixed tests, updated docs image --- bbot/modules/output/json.py | 4 ++-- bbot/test/test_step_1/test_python_api.py | 2 +- docs/scanning/output.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index 5b1c95195..e37a3829d 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -6,7 +6,7 @@ class JSON(BaseOutputModule): watched_events = ["*"] - meta = {"description": "Output to NDJSON"} + meta = {"description": "Output to Newline-Delimited JSON (NDJSON)"} options = {"output_file": "", "console": False} options_desc = {"output_file": "Output to file", "console": "Output to console"} @@ -29,4 +29,4 @@ async def cleanup(self): async def report(self): if self._file is not None: - self.info(f"Saved NDJSON output to {self.output_file}") + self.info(f"Saved JSON output to {self.output_file}") diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index d2a9958f8..00ad2d972 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -31,7 +31,7 @@ async def test_python_api(bbot_config): assert "scan_logging_test" not in open(debug_log).read() scan_home = scan3.helpers.scans_dir / "scan_logging_test" - out_file = scan_home / "output.json" + out_file = scan_home / "output.ndjson" assert list(scan3.helpers.read_file(out_file)) scan_log = scan_home / "scan.log" debug_log = scan_home / "debug.log" diff --git a/docs/scanning/output.md b/docs/scanning/output.md index c6543309b..310b92288 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -1,7 +1,7 @@ # Output By default, BBOT saves its output in TXT, JSON, and CSV formats: -![image](https://github.com/blacklanternsecurity/bbot/assets/20261699/779207f4-1c2f-4f65-a132-794ca8bd2f8a) +![bbot output](https://github.com/blacklanternsecurity/bbot/assets/20261699/bb3da441-2682-408f-b955-19b268823b82) Every BBOT scan gets a unique and mildly-entertaining name like **`demonic_jimmy`**. Output for that scan, including scan stats and any web screenshots, etc., are saved to a folder by that name in `~/.bbot/scans`. The most recent 20 scans are kept, and older ones are removed. You can change the location of BBOT's output with `--output`, and you can also pick a custom scan name with `--name`.