commenting base module

bbot/core/helpers/web.py

@@ -393,6 +393,18 @@ async def curl(self, *args, **kwargs):
         output = (await self.parent_helper.run(curl_command)).stdout
         return output
 
+    def is_spider_danger(self, source_event, url):
+        """
+        Todo: write tests for this
+        """
+        url_depth = self.parent_helper.url_depth(url)
+        web_spider_depth = self.parent_helper.scan.config.get("web_spider_depth", 1)
+        spider_distance = getattr(source_event, "web_spider_distance", 0) + 1
+        web_spider_distance = self.parent_helper.scan.config.get("web_spider_distance", 0)
+        if (url_depth > web_spider_depth) or (spider_distance > web_spider_distance):
+            return True
+        return False
+
 
 user_keywords = [re.compile(r, re.I) for r in ["user", "login", "email"]]
 pass_keywords = [re.compile(r, re.I) for r in ["pass"]]

bbot/modules/gowitness.py

@@ -148,7 +148,7 @@ async def handle_batch(self, *events):
             _id = row["url_id"]
             source_url = self.screenshots_taken[_id]
             source_event = events[source_url]
-            if self.is_spider_danger(source_event, url):
+            if self.helpers.is_spider_danger(source_event, url):
                 tags.append("spider-danger")
             if url and url.startswith("http"):
                 self.emit_event(url, "URL_UNVERIFIED", source=source_event, tags=tags)

bbot/modules/internal/excavate.py

@@ -78,7 +78,7 @@ async def search(self, content, event, **kwargs):
             url_event = self.report(result, name, event, **kwargs)
             if url_event is not None:
                 url_in_scope = self.excavate.scan.in_scope(url_event)
-                is_spider_danger = self.excavate.is_spider_danger(event, result)
+                is_spider_danger = self.excavate.helpers.is_spider_danger(event, result)
                 if (
                     (
                         urls_found >= self.web_spider_links_per_page and url_in_scope

bbot/modules/robots.py

@@ -46,6 +46,6 @@ async def handle_event(self, event):
                             continue
 
                         tags = []
-                        if self.is_spider_danger(event, unverified_url):
+                        if self.helpers.is_spider_danger(event, unverified_url):
                             tags.append("spider-danger")
                         self.emit_event(unverified_url, "URL_UNVERIFIED", source=event, tags=tags)

bbot/scanner/scanner.py

@@ -345,7 +345,7 @@ async def async_start(self):
                     break
 
                 if "python" in self.modules:
-                    events, finish = await self.modules["python"].events_waiting()
+                    events, finish = await self.modules["python"]._events_waiting()
                     for e in events:
                         yield e
 

docs/dev/scanner.md

@@ -1,3 +1 @@
-# `bbot.scanner.Scanner()`
-
 ::: bbot.scanner.Scanner

mkdocs.yml

@@ -32,6 +32,8 @@ nav:
     - How to Write a Module: contribution.md
     - Developer Reference:
       - Scanner: dev/scanner.md
+      - Target: dev/target.md
+      - BaseModule: dev/basemodule.md
       - Helpers: 
         # dev/helpers/index.md
         - Miscellaneous: dev/helpers/misc.md
@@ -60,6 +62,7 @@ plugins:
       handlers:
         python:
           options:
+            heading_level: 1
             show_signature_annotations: true
             show_root_toc_entry: false
             show_root_heading: true