From 4c732c56c2d2acf53f5643efb71860c529895283 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 19 Nov 2024 10:40:52 -0500 Subject: [PATCH 1/5] more precise cloud tags --- bbot/modules/internal/cloudcheck.py | 23 ++++++++++++++++--- bbot/test/bbot_fixtures.py | 14 +++++------ bbot/test/conftest.py | 2 +- .../module_tests/test_module_cloudcheck.py | 4 ++++ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 392c8e0c5..49d507330 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -1,3 +1,5 @@ +from contextlib import suppress + from bbot.modules.base import BaseInterceptModule @@ -29,12 +31,27 @@ async def handle_event(self, event, **kwargs): self.make_dummy_modules() # cloud tagging by hosts hosts_to_check = set(str(s) for s in event.resolved_hosts) - # we use the original host, since storage buckets hostnames might be collapsed to _wildcard - hosts_to_check.add(str(event.host_original)) + event_host = event.host_original + event_is_ip = self.helpers.is_ip(event_host) + with suppress(KeyError): + hosts_to_check.remove(event_host) + for provider, provider_type, subnet in self.helpers.cloudcheck(event_host): + if provider: + event.add_tag(f"{provider_type}-{provider}") + if event_is_ip: + event.add_tag(f"{provider_type}-ip") + else: + event.add_tag(f"{provider_type}-domain") + for host in hosts_to_check: + host_is_ip = self.helpers.is_ip(host) for provider, provider_type, subnet in self.helpers.cloudcheck(host): if provider: event.add_tag(f"{provider_type}-{provider}") + if host_is_ip: + event.add_tag(f"{provider_type}-ip") + elif not event_is_ip: + event.add_tag(f"{provider_type}-cname") found = set() # look for cloud assets in hosts, http responses @@ -54,7 +71,7 @@ async def handle_event(self, event, **kwargs): if event.type == "HTTP_RESPONSE": matches = await self.helpers.re.findall(sig, event.data.get("body", "")) elif event.type.startswith("DNS_NAME"): - for host in hosts_to_check: + for host in hosts_to_check.union([event_host]): match = sig.match(host) if match: matches.append(match.groups()) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index e1e3aa1b8..94df335a8 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -224,12 +224,12 @@ class bbot_events: return bbot_events -@pytest.fixture(scope="session", autouse=True) -def install_all_python_deps(): - deps_pip = set() - for module in DEFAULT_PRESET.module_loader.preloaded().values(): - deps_pip.update(set(module.get("deps", {}).get("pip", []))) +# @pytest.fixture(scope="session", autouse=True) +# def install_all_python_deps(): +# deps_pip = set() +# for module in DEFAULT_PRESET.module_loader.preloaded().values(): +# deps_pip.update(set(module.get("deps", {}).get("pip", []))) - constraint_file = tempwordlist(get_python_constraints()) +# constraint_file = tempwordlist(get_python_constraints()) - subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) +# subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 93d635e42..3a4901b12 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -337,7 +337,7 @@ def pytest_sessionfinish(session, exitstatus): logger.removeHandler(handler) # Wipe out BBOT home dir - shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) + # shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) yield diff --git a/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py b/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py index b95e7455d..902f2df35 100644 --- a/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +++ b/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py @@ -51,6 +51,10 @@ async def setup_after_prep(self, module_test): await module.handle_event(event) assert "cloud-amazon" in event.tags, f"{event} was not properly cloud-tagged" + assert "cloud-domain" in aws_event1.tags + assert "cloud-ip" in other_event2.tags + assert "cloud-cname" in other_event3.tags + for event in (aws_event3, other_event1): await module.handle_event(event) assert "cloud-amazon" not in event.tags, f"{event} was improperly cloud-tagged" From bbee6705e3540681b941a71f3ba70c7657400b3f Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 19 Nov 2024 10:41:24 -0500 Subject: [PATCH 2/5] blacked --- bbot/test/bbot_fixtures.py | 14 +++++++------- bbot/test/conftest.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 94df335a8..e1e3aa1b8 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -224,12 +224,12 @@ class bbot_events: return bbot_events -# @pytest.fixture(scope="session", autouse=True) -# def install_all_python_deps(): -# deps_pip = set() -# for module in DEFAULT_PRESET.module_loader.preloaded().values(): -# deps_pip.update(set(module.get("deps", {}).get("pip", []))) +@pytest.fixture(scope="session", autouse=True) +def install_all_python_deps(): + deps_pip = set() + for module in DEFAULT_PRESET.module_loader.preloaded().values(): + deps_pip.update(set(module.get("deps", {}).get("pip", []))) -# constraint_file = tempwordlist(get_python_constraints()) + constraint_file = tempwordlist(get_python_constraints()) -# subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) + subprocess.run([sys.executable, "-m", "pip", "install", "--constraint", constraint_file] + list(deps_pip)) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 3a4901b12..93d635e42 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -337,7 +337,7 @@ def pytest_sessionfinish(session, exitstatus): logger.removeHandler(handler) # Wipe out BBOT home dir - # shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) + shutil.rmtree("/tmp/.bbot_test", ignore_errors=True) yield From 5041c166730a313f4c7d90ea5146f89e3e7c9905 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 19 Nov 2024 18:48:02 -0500 Subject: [PATCH 3/5] organize --- bbot/modules/internal/cloudcheck.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 49d507330..285d48188 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -30,28 +30,25 @@ async def handle_event(self, event, **kwargs): if self.dummy_modules is None: self.make_dummy_modules() # cloud tagging by hosts - hosts_to_check = set(str(s) for s in event.resolved_hosts) - event_host = event.host_original - event_is_ip = self.helpers.is_ip(event_host) + hosts_to_check = set(event.resolved_hosts) with suppress(KeyError): - hosts_to_check.remove(event_host) - for provider, provider_type, subnet in self.helpers.cloudcheck(event_host): - if provider: - event.add_tag(f"{provider_type}-{provider}") - if event_is_ip: - event.add_tag(f"{provider_type}-ip") - else: - event.add_tag(f"{provider_type}-domain") + hosts_to_check.remove(event.host_original) + hosts_to_check = [event.host_original] + list(hosts_to_check) - for host in hosts_to_check: + for i, host in enumerate(hosts_to_check): host_is_ip = self.helpers.is_ip(host) for provider, provider_type, subnet in self.helpers.cloudcheck(host): if provider: event.add_tag(f"{provider_type}-{provider}") if host_is_ip: event.add_tag(f"{provider_type}-ip") - elif not event_is_ip: - event.add_tag(f"{provider_type}-cname") + else: + # if the original hostname is a cloud domain, tag it as such + if i == 0: + event.add_tag(f"{provider_type}-domain") + # any children are tagged as CNAMEs + else: + event.add_tag(f"{provider_type}-cname") found = set() # look for cloud assets in hosts, http responses @@ -71,7 +68,7 @@ async def handle_event(self, event, **kwargs): if event.type == "HTTP_RESPONSE": matches = await self.helpers.re.findall(sig, event.data.get("body", "")) elif event.type.startswith("DNS_NAME"): - for host in hosts_to_check.union([event_host]): + for host in hosts_to_check: match = sig.match(host) if match: matches.append(match.groups()) From 87b9f75276eafe54bbaa294090887638512bda9a Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 19 Nov 2024 18:48:31 -0500 Subject: [PATCH 4/5] fix portscan tests --- .../test_step_2/module_tests/test_module_portscan.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py index 56536cb5d..d9f55c27f 100644 --- a/bbot/test/test_step_2/module_tests/test_module_portscan.py +++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py @@ -109,10 +109,12 @@ def check(self, module_test, events): if e.type == "DNS_NAME" and e.data == "dummy.asdf.evilcorp.net" and str(e.module) == "dummy_module" ] ) - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 3 - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 3 - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 3 - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 3 + # the reason these numbers aren't exactly predictable is because we can't predict which one arrives first + # to the portscan module. Sometimes, one that would normally be deduped is force-emitted because it led to a new open port. + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 4 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 4 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 4 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 4 assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.5:80"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.6:631"]) From 0fd7102d3b55b751c943fdee5e62f1a7ec6c8b41 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 20 Nov 2024 11:36:11 -0500 Subject: [PATCH 5/5] fix cloudcheck bug --- bbot/modules/internal/cloudcheck.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 285d48188..42c51ec03 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -51,6 +51,7 @@ async def handle_event(self, event, **kwargs): event.add_tag(f"{provider_type}-cname") found = set() + str_hosts_to_check = [str(host) for host in hosts_to_check] # look for cloud assets in hosts, http responses # loop through each provider for provider in self.helpers.cloud.providers.values(): @@ -68,7 +69,7 @@ async def handle_event(self, event, **kwargs): if event.type == "HTTP_RESPONSE": matches = await self.helpers.re.findall(sig, event.data.get("body", "")) elif event.type.startswith("DNS_NAME"): - for host in hosts_to_check: + for host in str_hosts_to_check: match = sig.match(host) if match: matches.append(match.groups())