more tests

bbot/scanner/target.py

@@ -116,11 +116,11 @@ def _add(self, host, data):
         self.events.add(data)
         if host:
             try:
-                event_set = self.get(host, single=False, raise_error=True)
+                event_set = self.get(host, raise_error=True, single=False)
                 event_set.add(data)
             except KeyError:
                 event_set = {data}
-                super()._add(host, data=event_set)
+            super()._add(host, data=event_set)
 
     def check_special_target_types(self, target):
         for regex, callback in self.special_target_types.items():
@@ -193,14 +193,14 @@ def handle_regex(self, match):
         self.blacklist_regexes.add(blacklist_regex)
         return []
 
-    def get(self, event, **kwargs):
+    def get(self, event, single=True, **kwargs):
         """
         Here, for the blacklist, we modify this method to also consider any special regex patterns specified by the user
         """
         event = self.make_event(event)
         # first, check event's host against blacklist
         try:
-            event_result = super().get(event, raise_error=True)
+            event_result = super().get(event, raise_error=True, single=False)
         except KeyError:
             event_result = None
         if event_result is not None:

bbot/test/bbot_fixtures.py

@@ -224,9 +224,9 @@ class bbot_events:
     return bbot_events
 
 
-@pytest.fixture(scope="session", autouse=True)
-def install_all_python_deps():
-    deps_pip = set()
-    for module in DEFAULT_PRESET.module_loader.preloaded().values():
-        deps_pip.update(set(module.get("deps", {}).get("pip", [])))
-    subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))
+# @pytest.fixture(scope="session", autouse=True)
+# def install_all_python_deps():
+#     deps_pip = set()
+#     for module in DEFAULT_PRESET.module_loader.preloaded().values():
+#         deps_pip.update(set(module.get("deps", {}).get("pip", [])))
+#     subprocess.run([sys.executable, "-m", "pip", "install"] + list(deps_pip))

bbot/test/conftest.py

@@ -326,7 +326,7 @@ def pytest_sessionfinish(session, exitstatus):
             logger.removeHandler(handler)
 
     # Wipe out BBOT home dir
-    shutil.rmtree("/tmp/.bbot_test", ignore_errors=True)
+    # shutil.rmtree("/tmp/.bbot_test", ignore_errors=True)
 
     yield
 

bbot/test/test_step_1/test_target.py

@@ -371,11 +371,16 @@ async def test_blacklist_regex(bbot_scanner, bbot_httpserver):
     assert "http://test.com/asdf/123456.aspx?a=asdf" in blacklist
     assert "http://test.com/asdf/123456.aspx" in blacklist
 
-    bbot_httpserver.expect_request(uri="/").respond_with_data("<a href='http://127.0.0.1:8888/asdfevil333asdf'/>")
+    bbot_httpserver.expect_request(uri="/").respond_with_data("""
+        <a href='http://127.0.0.1:8888/asdfevil333asdf'/>
+        <a href='http://127.0.0.1:8888/logout.aspx'/>
+    """)
     bbot_httpserver.expect_request(uri="/asdfevilasdf").respond_with_data("")
+    bbot_httpserver.expect_request(uri="/logout.aspx").respond_with_data("")
 
     # make sure URL is detected normally
     scan = bbot_scanner("http://127.0.0.1:8888/", presets=["spider"], config={"excavate": True}, debug=True)
+    assert set([r.pattern for r in scan.target.blacklist.blacklist_regexes]) == {r"/.*(sign[_-]?out|log[_-]?out)"}
     events = [e async for e in scan.async_start()]
     urls = [e.data for e in events if e.type == "URL"]
     assert len(urls) == 2
@@ -391,7 +396,7 @@ async def test_blacklist_regex(bbot_scanner, bbot_httpserver):
     )
     print(scan.target.blacklist.blacklist_regexes)
     assert scan.target.blacklist.blacklist_regexes
-    assert next(iter(scan.target.blacklist.blacklist_regexes)).pattern == "evil[0-9]{3}"
+    assert set([r.pattern for r in scan.target.blacklist.blacklist_regexes]) == {r"evil[0-9]{3}", r"/.*(sign[_-]?out|log[_-]?out)"}
     events = [e async for e in scan.async_start()]
     urls = [e.data for e in events if e.type == "URL"]
     assert len(urls) == 1