Merge pull request #1042 from blacklanternsecurity/fix-event-stats

Fix event stats

bbot/modules/base.py

@@ -718,10 +718,6 @@ async def __event_postcheck(self, event):
             if not filter_result:
                 return False, msg
 
-        if self._type == "output" and not event._stats_recorded:
-            event._stats_recorded = True
-            self.scan.stats.event_produced(event)
-
         self.debug(f"{event} passed post-check")
         return True, ""
 

bbot/modules/output/base.py

@@ -41,6 +41,14 @@ def _event_precheck(self, event):
 
         return True, "precheck succeeded"
 
+    async def _event_postcheck(self, event):
+        acceptable, reason = await super()._event_postcheck(event)
+        if acceptable and not event._stats_recorded and event.type not in ("FINISHED",):
+            event._stats_recorded = True
+            self.scan.stats.event_distributed(event)
+            self.scan.stats.event_produced(event)
+        return acceptable, reason
+
     def is_incoming_duplicate(self, event, add=False):
         is_incoming_duplicate, reason = super().is_incoming_duplicate(event, add=add)
         # make exception for graph-important events

bbot/modules/sslcert.py

@@ -17,7 +17,7 @@ class sslcert(BaseModule):
     options = {"timeout": 5.0, "skip_non_ssl": True}
     options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"}
     deps_apt = ["openssl"]
-    deps_pip = ["pyOpenSSL~=23.1.1"]
+    deps_pip = ["pyOpenSSL~=24.0.0"]
     _max_event_handlers = 25
     scope_distance_modifier = 1
     _priority = 2

bbot/scanner/manager.py

@@ -412,8 +412,6 @@ async def distribute_event(self, event):
                 if acceptable_dup or graph_important:
                     await mod.queue_event(event)
 
-            self.scan.stats.event_distributed(event)
-
     async def _worker_loop(self):
         try:
             while not self.scan.stopped:

bbot/scanner/stats.py

@@ -15,7 +15,6 @@ def __init__(self, scan):
         self.scan = scan
         self.module_stats = {}
         self.events_emitted_by_type = {}
-        self.perf_stats = []
 
     def event_distributed(self, event):
         _increment(self.events_emitted_by_type, event.type)
@@ -66,9 +65,6 @@ def table(self):
         return [header] + table
 
     def _make_table(self):
-        self.perf_stats.sort(key=lambda x: x[-1])
-        for callback, runtime in self.perf_stats:
-            log.info(f"{callback}\t{runtime}")
         table = self.table()
         if len(table) == 1:
             table += [["None", "None", "None"]]

bbot/test/test_step_2/module_tests/test_module_aggregate.py

@@ -2,7 +2,10 @@
 
 
 class TestAggregate(ModuleTestBase):
-    config_overrides = {"dns_resolution": True}
+    config_overrides = {"dns_resolution": True, "scope_report_distance": 1}
+
+    async def setup_before_prep(self, module_test):
+        module_test.scan.helpers.dns.mock_dns({("blacklanternsecurity.com", "A"): "1.2.3.4"})
 
     def check(self, module_test, events):
         filename = next(module_test.scan.home.glob("scan-stats-table*.txt"))

pyproject.toml

@@ -45,7 +45,6 @@ aioconsole = "^0.6.2"
 pydantic = "^2.4.2"
 httpx = "^0.26.0"
 cloudcheck = "^2.1.0.181"
-pytest-httpx = "^0.28.0"
 tldextract = "^5.1.1"
 
 [tool.poetry.group.dev.dependencies]
@@ -62,6 +61,7 @@ pytest-timeout = "^2.1.0"
 pytest = "^7.4.0"
 pre-commit = "^3.4.0"
 black = "^24.1.1"
+pytest-httpx = "^0.29.0"
 
 [tool.poetry.group.docs.dependencies]
 mkdocs = "^1.5.2"