From 28bd86b06153dc35f5a3232e1607a58ebfd962b2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 21:51:22 -0500 Subject: [PATCH 1/5] fix excavate bug --- bbot/core/event/base.py | 28 +++++++++++++----------- bbot/modules/base.py | 2 +- bbot/test/bbot_fixtures.py | 12 +++++------ bbot/test/test_step_1/test_events.py | 32 ++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 30089dcc0..9f2655dcd 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1003,13 +1003,15 @@ def __init__(self, *args, **kwargs): if parent_url is not None: self.data["url"] = parent_url.geturl() # inherit closest path - if not "path" in self.data and isinstance(parent.data, dict): + if not "path" in self.data and isinstance(parent.data, dict) and not parent.type == "HTTP_RESPONSE": parent_path = parent.data.get("path", None) if parent_path is not None: self.data["path"] = parent_path # inherit closest host if parent.host: self.data["host"] = str(parent.host) + # we do this to refresh the hash + self.data = self.data break # die if we still haven't found a host if not self.host: @@ -1559,6 +1561,8 @@ def __init__(self, *args, **kwargs): self.add_tag("compressed") self.add_tag(f"{compression}-archive") self.data["compression"] = compression + # refresh hash + self.data = self.data class RAW_DNS_RECORD(DictHostEvent, DnsEvent): @@ -1639,23 +1643,23 @@ def make_event( tags = set(tags) if is_event(data): - data = copy(data) - if scan is not None and not data.scan: - data.scan = scan - if scans is not None and not data.scans: - data.scans = scans + event = copy(data) + if scan is not None and not event.scan: + event.scan = scan + if scans is not None and not event.scans: + event.scans = scans if module is not None: - data.module = module + event.module = module if parent is not None: - data.parent = parent + event.parent = parent if context is not None: - data.discovery_context = context + event.discovery_context = context if internal == True: - data.internal = True + event.internal = True if tags: - data.tags = tags.union(data.tags) + event.add_tags(tags) event_type = data.type - return data + return event else: if event_type is None: event_type, data = get_event_type(data) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 956d59c98..ef85f15e2 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1628,7 +1628,7 @@ async def _worker(self): forward_event, forward_event_reason = forward_event if forward_event is False: - self.debug(f"Not forwarding {event} because {forward_event_reason}") + self.debug(f"Not forwarding {event.type}:{event.data} because {forward_event_reason}") continue self.debug(f"Forwarding {event}") diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 0b2a0ec57..4860ea051 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -224,9 +224,9 @@ class bbot_events: return bbot_events -@pytest.fixture(scope="session", autouse=True) -def install_all_python_deps(): - deps_pip = set() - for module in DEFAULT_PRESET.module_loader.preloaded().values(): - deps_pip.update(set(module.get("deps", {}).get("pip", []))) - subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) +# @pytest.fixture(scope="session", autouse=True) +# def install_all_python_deps(): +# deps_pip = set() +# for module in DEFAULT_PRESET.module_loader.preloaded().values(): +# deps_pip.update(set(module.get("deps", {}).get("pip", []))) +# subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 1ebb38fea..3eeb6670b 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -966,3 +966,35 @@ def test_event_magic(): assert event.tags == {"folder"} zip_file.unlink() + + +def test_event_hashing(): + scan = Scanner("example.com") + url_event = scan.make_event("https://api.example.com/", "URL_UNVERIFIED", parent=scan.root_event) + host_event_1 = scan.make_event("www.example.com", "DNS_NAME", parent=url_event) + host_event_2 = scan.make_event("test.example.com", "DNS_NAME", parent=url_event) + finding_data = {"description": "Custom Yara Rule [find_string] Matched via identifier [str1]"} + finding1 = scan.make_event(finding_data, "FINDING", parent=host_event_1) + finding2 = scan.make_event(finding_data, "FINDING", parent=host_event_2) + finding3 = scan.make_event(finding_data, "FINDING", parent=host_event_2) + + assert finding1.data == { + "description": "Custom Yara Rule [find_string] Matched via identifier [str1]", + "host": "www.example.com", + } + assert finding2.data == { + "description": "Custom Yara Rule [find_string] Matched via identifier [str1]", + "host": "test.example.com", + } + assert finding3.data == { + "description": "Custom Yara Rule [find_string] Matched via identifier [str1]", + "host": "test.example.com", + } + assert finding1.id != finding2.id + assert finding2.id == finding3.id + assert finding1.data_id != finding2.data_id + assert finding2.data_id == finding3.data_id + assert finding1.data_hash != finding2.data_hash + assert finding2.data_hash == finding3.data_hash + assert hash(finding1) != hash(finding2) + assert hash(finding2) == hash(finding3) From e99daeef8244c22f8a82de3f193e8d24430f540e Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 21:59:08 -0500 Subject: [PATCH 2/5] uncomment --- bbot/test/bbot_fixtures.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 4860ea051..0b2a0ec57 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -224,9 +224,9 @@ class bbot_events: return bbot_events -# @pytest.fixture(scope="session", autouse=True) -# def install_all_python_deps(): -# deps_pip = set() -# for module in DEFAULT_PRESET.module_loader.preloaded().values(): -# deps_pip.update(set(module.get("deps", {}).get("pip", []))) -# subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) +@pytest.fixture(scope="session", autouse=True) +def install_all_python_deps(): + deps_pip = set() + for module in DEFAULT_PRESET.module_loader.preloaded().values(): + deps_pip.update(set(module.get("deps", {}).get("pip", []))) + subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip)) From ad22dfe157dc2f13f85596bdf5c9727044a90f8c Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 22:15:47 -0500 Subject: [PATCH 3/5] fix event tests --- bbot/core/event/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index 9f2655dcd..d185b1d74 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1657,7 +1657,7 @@ def make_event( if internal == True: event.internal = True if tags: - event.add_tags(tags) + event.tags = tags.union(event.tags) event_type = data.type return event else: From d71e34a8ce54ecdbd7d2513b958b0910a359aeba Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 22:50:07 -0500 Subject: [PATCH 4/5] fix tests --- bbot/modules/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index ef85f15e2..a9d64e1c0 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1628,7 +1628,7 @@ async def _worker(self): forward_event, forward_event_reason = forward_event if forward_event is False: - self.debug(f"Not forwarding {event.type}:{event.data} because {forward_event_reason}") + self.debug(f"Not forwarding {event.type} because {forward_event_reason}") continue self.debug(f"Forwarding {event}") From 9d48030ddd150a514c78dbd75053dae5b9d255f7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 23:10:21 -0500 Subject: [PATCH 5/5] fix tests again --- bbot/modules/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index a9d64e1c0..956d59c98 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -1628,7 +1628,7 @@ async def _worker(self): forward_event, forward_event_reason = forward_event if forward_event is False: - self.debug(f"Not forwarding {event.type} because {forward_event_reason}") + self.debug(f"Not forwarding {event} because {forward_event_reason}") continue self.debug(f"Forwarding {event}")