ensure deduping works as intended

bbot/modules/output/emails.py

@@ -14,9 +14,6 @@ class Emails(Human):
 
     output_filename = "emails.txt"
 
-    async def setup(self):
-        return await super().setup()
-
     def _scope_distance_check(self, event):
         return BaseModule._scope_distance_check(self, event)
 
@@ -25,11 +22,6 @@ async def handle_event(self, event):
             self.file.write(f"{event.data}\n")
             self.file.flush()
 
-    async def cleanup(self):
-        if getattr(self, "_file", None) is not None:
-            with suppress(Exception):
-                self.file.close()
-
     async def report(self):
         if getattr(self, "_file", None) is not None:
             self.info(f"Saved email addresses to {self.output_file}")

bbot/modules/output/subdomains.py

@@ -32,11 +32,6 @@ async def handle_event(self, event):
             self.file.write(f"{event.data}\n")
             self.file.flush()
 
-    async def cleanup(self):
-        if getattr(self, "_file", None) is not None:
-            with suppress(Exception):
-                self.file.close()
-
     async def report(self):
         if getattr(self, "_file", None) is not None:
             self.info(f"Saved subdomains to {self.output_file}")

bbot/test/test_step_2/module_tests/test_module_emails.py

@@ -2,15 +2,22 @@
 
 
 class TestEmais(ModuleTestBase):
-    modules_overrides = ["emails", "emailformat"]
+    modules_overrides = ["emails", "emailformat", "skymem"]
 
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(
             url="https://www.email-format.com/d/blacklanternsecurity.com/",
-            text="<p>[email protected]</a>",
+            text="<p>[email protected]</p>",
+        )
+        module_test.httpx_mock.add_response(
+            url="https://www.skymem.info/srch?q=blacklanternsecurity.com",
+            text="<p>[email protected]</p>",
         )
 
     def check(self, module_test, events):
-        sub_file = module_test.scan.home / "emails.txt"
-        emails = set(open(sub_file).read().splitlines())
-        assert emails == {"[email protected]"}
+        assert 2 == len([e for e in events if e.data == "[email protected]"])
+        email_file = module_test.scan.home / "emails.txt"
+        emails = open(email_file).read().splitlines()
+        # make sure deduping works as intended
+        assert len(emails) == 1
+        assert set(emails) == {"[email protected]"}