diff --git a/.gitattributes b/.gitattributes index 49edcb7119..00bf2637dc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,4 +5,4 @@ *.txt text eol=lf *.json text eol=lf *.md text eol=lf -*.sh text eol=lf \ No newline at end of file +*.sh text eol=lf diff --git a/.gitmodules b/.gitmodules index 0033a29676..c85f090f5f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "bbot/modules/playground"] path = bbot/modules/playground url = https://github.com/blacklanternsecurity/bbot-module-playground - branch = main \ No newline at end of file + branch = main diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..d6643f2ad3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +# Learn more about this config here: https://pre-commit.com/ + +# To enable these pre-commit hooks run: +# `pipx install pre-commit` or `brew install pre-commit` +# Then in the project root directory run `pre-commit install` + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-builtin-literals + - id: check-byte-order-marker + - id: check-case-conflict + # - id: check-docstring-first + # - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + # - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-toml + - id: check-vcs-permalinks + - id: check-xml + # - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + # - id: detect-private-key + - id: end-of-file-fixer + - id: file-contents-sorter + - id: fix-byte-order-marker + - id: forbid-new-submodules + - id: forbid-submodules + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: sort-simple-yaml + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.8.0 + hooks: + - id: ruff + - id: ruff-format + + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.23 + hooks: + - id: validate-pyproject diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 76802dac81..05f1a91271 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -814,7 +814,7 @@ def json(self, mode="json"): if parent_uuid: j["parent_uuid"] = parent_uuid # tags - j.update({"tags": list(self.tags)}) + j.update({"tags": sorted(self.tags)}) # parent module if self.module: j.update({"module": str(self.module)}) diff --git a/bbot/defaults.yml b/bbot/defaults.yml index 63f5f7e68b..61638595a0 100644 --- a/bbot/defaults.yml +++ b/bbot/defaults.yml @@ -74,7 +74,7 @@ dns: web: # HTTP proxy - http_proxy: + http_proxy: # Web user-agent user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97 # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed) diff --git a/bbot/models/pydantic.py b/bbot/models/pydantic.py index 356ab2e44c..b7c5baae9b 100644 --- a/bbot/models/pydantic.py +++ b/bbot/models/pydantic.py @@ -50,7 +50,6 @@ def _indexed_fields(cls): ### EVENT ### - class Event(BBOTBaseModel): uuid: Annotated[str, "indexed", "unique"] id: Annotated[str, "indexed"] @@ -93,7 +92,6 @@ def get_data(self): ### SCAN ### - class Scan(BBOTBaseModel): id: Annotated[str, "indexed", "unique"] name: str @@ -117,7 +115,6 @@ def from_scan(cls, scan): ### TARGET ### - class Target(BBOTBaseModel): name: str = "Default Target" strict_scope: bool = False diff --git a/bbot/models/sql.py b/bbot/models/sql.py index 82ccdb1f6f..78465511f6 100644 --- a/bbot/models/sql.py +++ b/bbot/models/sql.py @@ -61,7 +61,6 @@ def __eq__(self, other): ### EVENT ### - class Event(BBOTBaseModel, table=True): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -106,7 +105,6 @@ def get_data(self): ### SCAN ### - class Scan(BBOTBaseModel, table=True): id: str = Field(primary_key=True) name: str @@ -121,7 +119,6 @@ class Scan(BBOTBaseModel, table=True): ### TARGET ### - class Target(BBOTBaseModel, table=True): name: str = "Default Target" strict_scope: bool = False diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 685d67f9d7..86b6130d71 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -57,7 +57,9 @@ async def handle_event(self, event, **kwargs): for provider in self.helpers.cloud.providers.values(): provider_name = provider.name.lower() base_kwargs = { - "parent": event, "tags": [f"{provider.provider_type}-{provider_name}"], "_provider": provider_name + "parent": event, + "tags": [f"{provider.provider_type}-{provider_name}"], + "_provider": provider_name, } # loop through the provider's regex signatures, if any for event_type, sigs in provider.signatures.items(): diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 15facec564..bdca0ea5c3 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -306,9 +306,7 @@ def get_dns_parent(self, event): @property def emit_raw_records(self): if self._emit_raw_records is None: - watching_raw_records = any( - "RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values() - ) + watching_raw_records = any("RAW_DNS_RECORD" in m.get_watched_events() for m in self.scan.modules.values()) omitted_event_types = self.scan.config.get("omit_event_types", []) omit_raw_records = "RAW_DNS_RECORD" in omitted_event_types self._emit_raw_records = watching_raw_records or not omit_raw_records diff --git a/bbot/modules/output/elastic.py b/bbot/modules/output/elastic.py index 15bc023df8..42c331c516 100644 --- a/bbot/modules/output/elastic.py +++ b/bbot/modules/output/elastic.py @@ -2,6 +2,10 @@ class Elastic(HTTP): + """ + docker run -d -p 9200:9200 --name=bbot-elastic --v "$(pwd)/elastic_data:/usr/share/elasticsearch/data" -e ELASTIC_PASSWORD=bbotislife -m 1GB docker.elastic.co/elasticsearch/elasticsearch:8.16.0 + """ + watched_events = ["*"] metadata = { "description": "Send scan results to Elasticsearch", @@ -9,9 +13,9 @@ class Elastic(HTTP): "author": "@TheTechromancer", } options = { - "url": "", + "url": "https://localhost:9200/bbot_events/_doc", "username": "elastic", - "password": "changeme", + "password": "bbotislife", "timeout": 10, } options_desc = { @@ -20,3 +24,9 @@ class Elastic(HTTP): "password": "Elastic password", "timeout": "HTTP timeout", } + + async def cleanup(self): + # refresh the index + doc_regex = self.helpers.re.compile(r"/[^/]+$") + refresh_url = doc_regex.sub("/_refresh", self.url) + await self.helpers.request(refresh_url, auth=self.auth) diff --git a/bbot/modules/output/kafka.py b/bbot/modules/output/kafka.py index 0a31e0be12..01eeeb2fd6 100644 --- a/bbot/modules/output/kafka.py +++ b/bbot/modules/output/kafka.py @@ -8,7 +8,7 @@ class Kafka(BaseOutputModule): watched_events = ["*"] meta = { "description": "Output scan data to a Kafka topic", - "created_date": "2024-11-17", + "created_date": "2024-11-22", "author": "@TheTechromancer", } options = { diff --git a/bbot/modules/output/mongo.py b/bbot/modules/output/mongo.py index 6ad16620f6..118ca82378 100644 --- a/bbot/modules/output/mongo.py +++ b/bbot/modules/output/mongo.py @@ -59,7 +59,13 @@ async def setup(self): async def handle_event(self, event): event_json = event.json() event_pydantic = Event(**event_json) - await self.events_collection.insert_one(event_pydantic.model_dump()) + while 1: + try: + await self.events_collection.insert_one(event_pydantic.model_dump()) + break + except Exception as e: + self.warning(f"Error inserting event into MongoDB: {e}, retrying...") + await self.helpers.sleep(1) if event.type == "SCAN": scan_json = Scan(**event.data_json).model_dump() diff --git a/bbot/modules/output/rabbitmq.py b/bbot/modules/output/rabbitmq.py new file mode 100644 index 0000000000..ba4205940d --- /dev/null +++ b/bbot/modules/output/rabbitmq.py @@ -0,0 +1,56 @@ +import json +import aio_pika + +from bbot.modules.output.base import BaseOutputModule + + +class RabbitMQ(BaseOutputModule): + watched_events = ["*"] + meta = { + "description": "Output scan data to a RabbitMQ queue", + "created_date": "2024-11-22", + "author": "@TheTechromancer", + } + options = { + "url": "amqp://guest:guest@localhost/", + "queue": "bbot_events", + } + options_desc = { + "url": "The RabbitMQ connection URL", + "queue": "The RabbitMQ queue to publish events to", + } + deps_pip = ["aio_pika~=9.5.0"] + + async def setup(self): + self.rabbitmq_url = self.config.get("url", "amqp://guest:guest@localhost/") + self.queue_name = self.config.get("queue", "bbot_events") + + # Connect to RabbitMQ + self.connection = await aio_pika.connect_robust(self.rabbitmq_url) + self.channel = await self.connection.channel() + + # Declare the queue + self.queue = await self.channel.declare_queue(self.queue_name, durable=True) + self.verbose("RabbitMQ connection and queue setup successfully") + return True + + async def handle_event(self, event): + event_json = event.json() + event_data = json.dumps(event_json).encode("utf-8") + + # Publish the message to the queue + while 1: + try: + await self.channel.default_exchange.publish( + aio_pika.Message(body=event_data), + routing_key=self.queue_name, + ) + break + except Exception as e: + self.error(f"Error publishing message to RabbitMQ: {e}, rerying...") + await self.helpers.sleep(1) + + async def cleanup(self): + # Close the connection + await self.connection.close() + self.verbose("RabbitMQ connection closed successfully") diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index 771e4b4f7f..3b3c488d15 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -207,7 +207,14 @@ async def get_asn_bgpview(self, ip): return False asns_tried.add(asn) asns.append( - {"asn": asn, "subnet": subnet, "name": name, "description": description, "country": country, "emails": emails} + { + "asn": asn, + "subnet": subnet, + "name": name, + "description": description, + "country": country, + "emails": emails, + } ) if not asns: self.debug(f'No results for "{ip}"') diff --git a/bbot/presets/kitchen-sink.yml b/bbot/presets/kitchen-sink.yml index 43057bf44a..073f480bb2 100644 --- a/bbot/presets/kitchen-sink.yml +++ b/bbot/presets/kitchen-sink.yml @@ -16,5 +16,3 @@ config: modules: baddns: enable_references: True - - diff --git a/bbot/presets/web/dotnet-audit.yml b/bbot/presets/web/dotnet-audit.yml index bbc5e201e0..b1cd8e9cac 100644 --- a/bbot/presets/web/dotnet-audit.yml +++ b/bbot/presets/web/dotnet-audit.yml @@ -19,4 +19,3 @@ config: extensions: asp,aspx,ashx,asmx,ascx telerik: exploit_RAU_crypto: True - diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index b4294710fa..d7b55d50c9 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -175,7 +175,9 @@ def preset_from_args(self): def create_parser(self, *args, **kwargs): kwargs.update( { - "description": "Bighuge BLS OSINT Tool", "formatter_class": argparse.RawTextHelpFormatter, "epilog": self.epilog + "description": "Bighuge BLS OSINT Tool", + "formatter_class": argparse.RawTextHelpFormatter, + "epilog": self.epilog, } ) p = argparse.ArgumentParser(*args, **kwargs) diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index 9e67f2c803..b275cc1f72 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -967,7 +967,7 @@ def presets_table(self, include_modules=True): header = ["Preset", "Category", "Description", "# Modules"] if include_modules: header.append("Modules") - for (loaded_preset, category, preset_path, original_file) in self.all_presets.values(): + for loaded_preset, category, preset_path, original_file in self.all_presets.values(): loaded_preset = loaded_preset.bake() num_modules = f"{len(loaded_preset.scan_modules):,}" row = [loaded_preset.name, category, loaded_preset.description, num_modules] diff --git a/bbot/test/test_step_1/test__module__tests.py b/bbot/test/test_step_1/test__module__tests.py index 791e58f58a..e50f67a910 100644 --- a/bbot/test/test_step_1/test__module__tests.py +++ b/bbot/test/test_step_1/test__module__tests.py @@ -15,7 +15,6 @@ def test__module__tests(): - preset = Preset() # make sure each module has a .py file diff --git a/bbot/test/test_step_1/test_bbot_fastapi.py b/bbot/test/test_step_1/test_bbot_fastapi.py index feaf8686da..9f54e4a881 100644 --- a/bbot/test/test_step_1/test_bbot_fastapi.py +++ b/bbot/test/test_step_1/test_bbot_fastapi.py @@ -17,7 +17,6 @@ def run_bbot_multiprocess(queue): def test_bbot_multiprocess(bbot_httpserver): - bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com") queue = multiprocessing.Queue() @@ -32,12 +31,10 @@ def test_bbot_multiprocess(bbot_httpserver): def test_bbot_fastapi(bbot_httpserver): - bbot_httpserver.expect_request("/").respond_with_data("test@blacklanternsecurity.com") fastapi_process = start_fastapi_server() try: - # wait for the server to start with a timeout of 60 seconds start_time = time.time() while True: diff --git a/bbot/test/test_step_1/test_bloom_filter.py b/bbot/test/test_step_1/test_bloom_filter.py index 22ec4db323..f954bfbc6e 100644 --- a/bbot/test/test_step_1/test_bloom_filter.py +++ b/bbot/test/test_step_1/test_bloom_filter.py @@ -6,7 +6,6 @@ @pytest.mark.asyncio async def test_bloom_filter(): - def generate_random_strings(n, length=10): """Generate a list of n random strings.""" return ["".join(random.choices(string.ascii_letters + string.digits, k=length)) for _ in range(n)] diff --git a/bbot/test/test_step_1/test_dns.py b/bbot/test/test_step_1/test_dns.py index dbbfe68d65..c032b44e48 100644 --- a/bbot/test/test_step_1/test_dns.py +++ b/bbot/test/test_step_1/test_dns.py @@ -185,7 +185,6 @@ async def test_dns_resolution(bbot_scanner): @pytest.mark.asyncio async def test_wildcards(bbot_scanner): - scan = bbot_scanner("1.1.1.1") helpers = scan.helpers @@ -634,7 +633,6 @@ def custom_lookup(query, rdtype): @pytest.mark.asyncio async def test_wildcard_deduplication(bbot_scanner): - custom_lookup = """ def custom_lookup(query, rdtype): if rdtype == "TXT" and query.strip(".").endswith("evilcorp.com"): @@ -670,7 +668,6 @@ async def handle_event(self, event): @pytest.mark.asyncio async def test_dns_raw_records(bbot_scanner): - from bbot.modules.base import BaseModule class DummyModule(BaseModule): diff --git a/bbot/test/test_step_1/test_engine.py b/bbot/test/test_step_1/test_engine.py index dbb21246f2..653c3dcd6c 100644 --- a/bbot/test/test_step_1/test_engine.py +++ b/bbot/test/test_step_1/test_engine.py @@ -14,7 +14,6 @@ async def test_engine(): return_errored = False class TestEngineServer(EngineServer): - CMDS = { 0: "return_thing", 1: "yield_stuff", @@ -54,7 +53,6 @@ async def yield_stuff(self, n): raise class TestEngineClient(EngineClient): - SERVER_CLASS = TestEngineServer async def return_thing(self, n): diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 6ee006217d..8e0e945482 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -9,7 +9,6 @@ @pytest.mark.asyncio async def test_events(events, helpers): - scan = Scanner() await scan._prep() @@ -606,7 +605,6 @@ async def test_events(events, helpers): @pytest.mark.asyncio async def test_event_discovery_context(): - from bbot.modules.base import BaseModule scan = Scanner("evilcorp.com") diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py index 16b0dc9ec5..2eb67cd13d 100644 --- a/bbot/test/test_step_1/test_helpers.py +++ b/bbot/test/test_step_1/test_helpers.py @@ -857,7 +857,6 @@ def test_liststring_invalidfnchars(helpers): # test parameter validation @pytest.mark.asyncio async def test_parameter_validation(helpers): - getparam_valid_params = { "name", "age", diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 73fdcf23a5..5b1564f12c 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -16,7 +16,7 @@ def test_preset_descriptions(): # ensure very preset has a description preset = Preset() - for (loaded_preset, category, preset_path, original_filename) in preset.all_presets.values(): + for loaded_preset, category, preset_path, original_filename in preset.all_presets.values(): assert ( loaded_preset.description ), f'Preset "{loaded_preset.name}" at {original_filename} does not have a description.' @@ -68,7 +68,6 @@ def test_core(): def test_preset_yaml(clean_default_config): - import yaml preset1 = Preset( @@ -171,7 +170,6 @@ def test_preset_cache(): def test_preset_scope(): - # test target merging scan = Scanner("1.2.3.4", preset=Preset.from_dict({"target": ["evilcorp.com"]})) assert {str(h) for h in scan.preset.target.seeds.hosts} == {"1.2.3.4/32", "evilcorp.com"} @@ -378,7 +376,6 @@ def test_preset_scope(): @pytest.mark.asyncio async def test_preset_logging(): - scan = Scanner() # test individual verbosity levels @@ -711,7 +708,6 @@ class TestModule5(BaseModule): def test_preset_include(): - # test recursive preset inclusion custom_preset_dir_1 = bbot_test_dir / "custom_preset_dir" @@ -883,7 +879,6 @@ def test_preset_module_disablement(clean_default_config): def test_preset_require_exclude(): - def get_module_flags(p): for m in p.scan_modules: preloaded = p.preloaded_module(m) diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py index 1a549d549e..e282b1cc4f 100644 --- a/bbot/test/test_step_1/test_python_api.py +++ b/bbot/test/test_step_1/test_python_api.py @@ -119,7 +119,7 @@ def test_python_api_validation(): # normal module as output module with pytest.raises(ValidationError) as error: Scanner(output_modules=["robots"]) - assert str(error.value) == 'Could not find output module "robots". Did you mean "nats"?' + assert str(error.value) == 'Could not find output module "robots". Did you mean "rabbitmq"?' # invalid preset type with pytest.raises(ValidationError) as error: Scanner(preset="asdf") diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 3c9a9832b5..8f2a6bf91f 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -337,7 +337,6 @@ async def test_target(bbot_scanner): @pytest.mark.asyncio async def test_blacklist_regex(bbot_scanner, bbot_httpserver): - from bbot.scanner.target import ScanBlacklist blacklist = ScanBlacklist("evilcorp.com") diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index dc1f50339e..e07ed3d7d4 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -6,7 +6,6 @@ @pytest.mark.asyncio async def test_web_engine(bbot_scanner, bbot_httpserver, httpx_mock): - from werkzeug.wrappers import Response def server_handler(request): @@ -134,7 +133,6 @@ def server_handler(request): @pytest.mark.asyncio async def test_web_helpers(bbot_scanner, bbot_httpserver, httpx_mock): - # json conversion scan = bbot_scanner("evilcorp.com") url = "http://www.evilcorp.com/json_test?a=b" diff --git a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py index 77a86153c7..b2b49717c8 100644 --- a/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py +++ b/bbot/test/test_step_2/module_tests/test_module_baddns_direct.py @@ -55,8 +55,8 @@ def set_target(self, target): def check(self, module_test, events): assert any( e.type == "FINDING" - and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" - in e.data["description"] - for e in events + and "Possible [AWS Bucket Takeover Detection] via direct BadDNS analysis. Indicator: [[Words: The specified bucket does not exist | Condition: and | Part: body] Matchers-Condition: and] Trigger: [self] baddns Module: [CNAME]" + in e.data["description"] + for e in events ), "Failed to emit FINDING" assert any("baddns-cname" in e.tags for e in events), "Failed to add baddns tag" diff --git a/bbot/test/test_step_2/module_tests/test_module_elastic.py b/bbot/test/test_step_2/module_tests/test_module_elastic.py index 710c22e0f0..db9f2359f7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_elastic.py +++ b/bbot/test/test_step_2/module_tests/test_module_elastic.py @@ -48,12 +48,11 @@ async def setup_before_prep(self, module_test): response.raise_for_status() break except Exception as e: - print(f"Connection failed: {e}. Retrying...", flush=True) + self.log.verbose(f"Connection failed: {e}. Retrying...") time.sleep(0.5) # Ensure the index is empty await client.delete(f"https://localhost:9200/bbot_test_events", auth=("elastic", "bbotislife")) - print("Elasticsearch index cleaned up", flush=True) async def check(self, module_test, events): try: @@ -65,17 +64,11 @@ async def check(self, module_test, events): # Connect to Elasticsearch async with httpx.AsyncClient(verify=False) as client: - # refresh the index - await client.post(f"https://localhost:9200/bbot_test_events/_refresh", auth=("elastic", "bbotislife")) - # Fetch all events from the index response = await client.get( f"https://localhost:9200/bbot_test_events/_search?size=100", auth=("elastic", "bbotislife") ) response_json = response.json() - import json - - print(f"response: {json.dumps(response_json, indent=2)}") db_events = [hit["_source"] for hit in response_json["hits"]["hits"]] # make sure we have the same number of events @@ -124,7 +117,7 @@ async def check(self, module_test, events): auth=("elastic", "bbotislife"), params={"ignore": "400,404"}, ) - print(f"Deleted documents from index", flush=True) + self.log.verbose(f"Deleted documents from index") await asyncio.create_subprocess_exec( "docker", "stop", "bbot-test-elastic", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py index f5f774e380..a2ccf97613 100644 --- a/bbot/test/test_step_2/module_tests/test_module_excavate.py +++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py @@ -895,7 +895,7 @@ class TestExcavateRAWTEXT(ModuleTestBase): /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans << ->> +>> /Type /Page >> endobj @@ -906,7 +906,7 @@ class TestExcavateRAWTEXT(ModuleTestBase): endobj 5 0 obj << -/Author (anonymous) /CreationDate (D:20240807182842+00'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20240807182842+00'00') /Producer (ReportLab PDF Library - www.reportlab.com) +/Author (anonymous) /CreationDate (D:20240807182842+00'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20240807182842+00'00') /Producer (ReportLab PDF Library - www.reportlab.com) /Subject (unspecified) /Title (untitled) /Trapped /False >> endobj @@ -924,17 +924,17 @@ class TestExcavateRAWTEXT(ModuleTestBase): endobj xref 0 8 -0000000000 65535 f -0000000073 00000 n -0000000104 00000 n -0000000211 00000 n -0000000414 00000 n -0000000482 00000 n -0000000778 00000 n -0000000837 00000 n +0000000000 65535 f +0000000073 00000 n +0000000104 00000 n +0000000211 00000 n +0000000414 00000 n +0000000482 00000 n +0000000778 00000 n +0000000837 00000 n trailer << -/ID +/ID [<3c7340500fa2fe72523c5e6f07511599><3c7340500fa2fe72523c5e6f07511599>] % ReportLab generated PDF document -- digest (http://www.reportlab.com) diff --git a/bbot/test/test_step_2/module_tests/test_module_gowitness.py b/bbot/test/test_step_2/module_tests/test_module_gowitness.py index 2d6dc2cd8f..6090fbb1d6 100644 --- a/bbot/test/test_step_2/module_tests/test_module_gowitness.py +++ b/bbot/test/test_step_2/module_tests/test_module_gowitness.py @@ -101,6 +101,4 @@ class TestGoWitnessWithBlob(TestGowitness): def check(self, module_test, events): webscreenshots = [e for e in events if e.type == "WEBSCREENSHOT"] assert webscreenshots, "failed to raise WEBSCREENSHOT events" - assert all( - "blob" in e.data and e.data["blob"] for e in webscreenshots - ), "blob not found in WEBSCREENSHOT data" + assert all("blob" in e.data and e.data["blob"] for e in webscreenshots), "blob not found in WEBSCREENSHOT data" diff --git a/bbot/test/test_step_2/module_tests/test_module_newsletters.py b/bbot/test/test_step_2/module_tests/test_module_newsletters.py index 98210f658e..c5edd25141 100644 --- a/bbot/test/test_step_2/module_tests/test_module_newsletters.py +++ b/bbot/test/test_step_2/module_tests/test_module_newsletters.py @@ -10,16 +10,16 @@ class TestNewsletters(ModuleTestBase): modules_overrides = ["speculate", "httpx", "newsletters"] html_with_newsletter = """ - """ diff --git a/bbot/test/test_step_2/module_tests/test_module_ntlm.py b/bbot/test/test_step_2/module_tests/test_module_ntlm.py index 1e79be7705..7b834ef2f9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_ntlm.py +++ b/bbot/test/test_step_2/module_tests/test_module_ntlm.py @@ -10,7 +10,8 @@ async def setup_after_prep(self, module_test): request_args = {"uri": "/", "headers": {"test": "header"}} module_test.set_expect_requests(request_args, {}) request_args = { - "uri": "/oab/", "headers": {"Authorization": "NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw=="} + "uri": "/oab/", + "headers": {"Authorization": "NTLM TlRMTVNTUAABAAAAl4II4gAAAAAAAAAAAAAAAAAAAAAKAGFKAAAADw=="}, } respond_args = { "headers": { diff --git a/bbot/test/test_step_2/module_tests/test_module_pgp.py b/bbot/test/test_step_2/module_tests/test_module_pgp.py index e6f122dd93..dc493d7b52 100644 --- a/bbot/test/test_step_2/module_tests/test_module_pgp.py +++ b/bbot/test/test_step_2/module_tests/test_module_pgp.py @@ -9,10 +9,10 @@ class TestPGP(ModuleTestBase):
Type bits/keyID cr. time exp time key expirdiff --git a/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py b/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py new file mode 100644 index 0000000000..d05808c2da --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py @@ -0,0 +1,69 @@ +import json +import asyncio +from contextlib import suppress + +from .base import ModuleTestBase + + +class TestRabbitMQ(ModuleTestBase): + config_overrides = { + "modules": { + "rabbitmq": { + "url": "amqp://guest:guest@localhost/", + "queue": "bbot_events", + } + } + } + skip_distro_tests = True + + async def setup_before_prep(self, module_test): + import aio_pika + + # Start RabbitMQ + await asyncio.create_subprocess_exec( + "docker", "run", "-d", "--rm", "--name", "bbot-test-rabbitmq", "-p", "5672:5672", "rabbitmq:3-management" + ) + + # Wait for RabbitMQ to be ready + while True: + try: + # Attempt to connect to RabbitMQ with a timeout + connection = await aio_pika.connect_robust("amqp://guest:guest@localhost/") + break # Exit the loop if the connection is successful + except Exception as e: + with suppress(Exception): + await connection.close() + self.log.verbose(f"Waiting for RabbitMQ to be ready: {e}") + await asyncio.sleep(0.5) # Wait a bit before retrying + + self.connection = connection + self.channel = await self.connection.channel() + self.queue = await self.channel.declare_queue("bbot_events", durable=True) + + async def check(self, module_test, events): + try: + events_json = [e.json() for e in events] + events_json.sort(key=lambda x: x["timestamp"]) + + # Collect events from RabbitMQ + rabbitmq_events = [] + async with self.queue.iterator() as queue_iter: + async for message in queue_iter: + async with message.process(): + event_data = json.loads(message.body.decode("utf-8")) + rabbitmq_events.append(event_data) + if len(rabbitmq_events) >= len(events_json): + break + + rabbitmq_events.sort(key=lambda x: x["timestamp"]) + + # Verify the events match + assert events_json == rabbitmq_events, "Events do not match" + + finally: + # Clean up: Close the RabbitMQ connection + await self.connection.close() + # Stop RabbitMQ container + await asyncio.create_subprocess_exec( + "docker", "stop", "bbot-test-rabbitmq", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) diff --git a/bbot/test/test_step_2/module_tests/test_module_smuggler.py b/bbot/test/test_step_2/module_tests/test_module_smuggler.py index dcbb9fd3b5..fb86b9ae92 100644 --- a/bbot/test/test_step_2/module_tests/test_module_smuggler.py +++ b/bbot/test/test_step_2/module_tests/test_module_smuggler.py @@ -1,13 +1,13 @@ from .base import ModuleTestBase smuggler_text = r""" - ______ _ - / _____) | | - ( (____ ____ _ _ ____ ____| | _____ ____ + ______ _ + / _____) | | + ( (____ ____ _ _ ____ ____| | _____ ____ \____ \| \| | | |/ _ |/ _ | || ___ |/ ___) - _____) ) | | | |_| ( (_| ( (_| | || ____| | - (______/|_|_|_|____/ \___ |\___ |\_)_____)_| - (_____(_____| + _____) ) | | | |_| ( (_| ( (_| | || ____| | + (______/|_|_|_|____/ \___ |\___ |\_)_____)_| + (_____(_____| @defparam v1.1 @@ -16,13 +16,13 @@ [+] Endpoint : / [+] Configfile : default.py [+] Timeout : 5.0 seconds - [+] Cookies : 1 (Appending to the attack) - [nameprefix1] : Checking TECL... - [nameprefix1] : Checking CLTE... - [nameprefix1] : OK (TECL: 0.61 - 405) (CLTE: 0.62 - 405) - [tabprefix1] : Checking TECL...git - [tabprefix1] : Checking CLTE... - [tabprefix1] : Checking TECL... + [+] Cookies : 1 (Appending to the attack) + [nameprefix1] : Checking TECL... + [nameprefix1] : Checking CLTE... + [nameprefix1] : OK (TECL: 0.61 - 405) (CLTE: 0.62 - 405) + [tabprefix1] : Checking TECL...git + [tabprefix1] : Checking CLTE... + [tabprefix1] : Checking TECL... [tabprefix1] : Checking CLTE... [tabprefix1] : Checking TECL... [tabprefix1] : Checking CLTE... diff --git a/bbot/test/test_step_2/module_tests/test_module_speculate.py b/bbot/test/test_step_2/module_tests/test_module_speculate.py index e407470346..55db777e7b 100644 --- a/bbot/test/test_step_2/module_tests/test_module_speculate.py +++ b/bbot/test/test_step_2/module_tests/test_module_speculate.py @@ -63,7 +63,7 @@ def check(self, module_test, events): events_data.add(e.data) assert all( x in events_data - for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") + for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") ) @@ -78,5 +78,5 @@ def check(self, module_test, events): events_data.add(e.data) assert not any( x in events_data - for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") + for x in ("evilcorp.com:80", "evilcorp.com:443", "asdf.evilcorp.com:80", "asdf.evilcorp.com:443") ) diff --git a/bbot/test/test_step_2/module_tests/test_module_viewdns.py b/bbot/test/test_step_2/module_tests/test_module_viewdns.py index d196981ba1..e8b2fe2339 100644 --- a/bbot/test/test_step_2/module_tests/test_module_viewdns.py +++ b/bbot/test/test_step_2/module_tests/test_module_viewdns.py @@ -66,7 +66,7 @@ def check(self, module_test, events):