Merge branch 'dev' into baddns_module

bbot/modules/bypass403.py

@@ -63,6 +63,7 @@
     "X-Host": "127.0.0.1",
 }
 
+# This is planned to be replaced in the future: https://github.com/blacklanternsecurity/bbot/issues/1068
 waf_strings = ["The requested URL was rejected"]
 
 for qp in query_payloads:
@@ -83,8 +84,13 @@ class bypass403(BaseModule):
 
     async def do_checks(self, compare_helper, event, collapse_threshold):
         results = set()
+        error_count = 0
 
         for sig in signatures:
+            if error_count > 3:
+                self.warning(f"Received too many errors for URL {event.data} aborting bypass403")
+                return None
+
             sig = self.format_signature(sig, event)
             if sig[2] != None:
                 headers = dict(sig[2])
@@ -95,6 +101,7 @@ async def do_checks(self, compare_helper, event, collapse_threshold):
                     sig[1], headers=headers, method=sig[0], allow_redirects=True
                 )
             except HttpCompareError as e:
+                error_count += 1
                 self.debug(e)
                 continue
 
@@ -149,6 +156,7 @@ async def handle_event(self, event):
                     source=event,
                 )
 
+    # When a WAF-check helper is available in the future, we will convert to HTTP_RESPONSE and check for the WAF string here.
     async def filter_event(self, event):
         if ("status-403" in event.tags) or ("status-401" in event.tags):
             return True

bbot/modules/deadly/nuclei.py

@@ -70,7 +70,7 @@ async def setup(self):
         else:
             self.warning("Error running nuclei template update command")
         self.proxy = self.scan.config.get("http_proxy", "")
-        self.mode = self.config.get("mode", "severe")
+        self.mode = self.config.get("mode", "severe").lower()
         self.ratelimit = int(self.config.get("ratelimit", 150))
         self.concurrency = int(self.config.get("concurrency", 25))
         self.budget = int(self.config.get("budget", 1))

bbot/modules/gowitness.py

@@ -94,6 +94,20 @@ async def setup(self):
         custom_chrome_path = self.helpers.tools_dir / "chrome-linux" / "chrome"
         if custom_chrome_path.is_file():
             self.chrome_path = custom_chrome_path
+
+        # make sure we have a working chrome install
+        chrome_test_pass = False
+        for binary in ("chrome", "chromium", custom_chrome_path):
+            binary_path = self.helpers.which(binary)
+            if binary_path and Path(binary_path).is_file():
+                chrome_test_proc = await self.helpers.run([binary_path, "--version"])
+                if getattr(chrome_test_proc, "returncode", 1) == 0:
+                    self.verbose(f"Found chrome executable at {binary_path}")
+                    chrome_test_pass = True
+                    break
+        if not chrome_test_pass:
+            return False, "Failed to set up Google chrome. Please install manually or try again with --force-deps."
+
         self.db_path = self.base_path / "gowitness.sqlite3"
         self.screenshot_path = self.base_path / "screenshots"
         self.command = self.construct_command()
@@ -200,6 +214,7 @@ def new_screenshots(self):
         if self.db_path.is_file():
             with sqlite3.connect(str(self.db_path)) as con:
                 con.row_factory = sqlite3.Row
+                con.text_factory = self.helpers.smart_decode
                 cur = con.cursor()
                 res = self.cur_execute(cur, "SELECT * FROM urls")
                 for row in res:

docs/modules/nuclei.md

@@ -8,7 +8,7 @@ BBOT integrates with [Nuclei](https://github.com/projectdiscovery/nuclei), an op
 
 
 * The BBOT Nuclei module ingests **[URL]** events and emits events of type **[VULNERABILITY]** or **[FINDING]**
-* Vulnerabilities will inherit their severity from the Nuclei templates​
+* Vulnerabilities will inherit their severity from the Nuclei templates
 * Nuclei templates of severity INFO will be emitted as **[FINDINGS]**
 
 ## Default Behavior
@@ -59,15 +59,15 @@ This is equivalent to the Nuclei '-as' scan option. It only use templates that m
 
 #### Budget
 
-Budget mode is unique to BBOT. ​
+Budget mode is unique to BBOT.
 
-For larger scans with thousands of targets, doing a FULL Nuclei scan (1000s of Requests) for each is not realistic. ​
-As an alternative to the other modes, you can take advantage of Nuclei's "collapsible" template feature. ​
+For larger scans with thousands of targets, doing a FULL Nuclei scan (1000s of Requests) for each is not realistic. 
+As an alternative to the other modes, you can take advantage of Nuclei's "collapsible" template feature. 
 
 For only the cost of one (or more) "extra" request(s) per host, it can activate several hundred modules. These are modules which happen to look at a BaseUrl, and typically look for a specific string or other attribute. Nuclei is smart about reusing the request data when it can, and we can use this to our advantage. 
 
-The budget parameter is the # of extra requests per host you are willing to send to "feed" Nuclei templates​ (defaults to 1).
-For those times when vulnerability scanning isn't the main focus, but you want to look for easy wins.​
+The budget parameter is the # of extra requests per host you are willing to send to "feed" Nuclei templates (defaults to 1).
+For those times when vulnerability scanning isn't the main focus, but you want to look for easy wins.
 
 Of course, there is a rapidly diminishing return when you set he value to more than a handful. Eventually, this becomes 1 template per 1 budget value increase. However, in the 1-10 range there is a lot of value. This graphic should give you a rough visual idea of this concept.
 
@@ -86,20 +86,20 @@ The **ratelimit** and **concurrency** settings default to the same defaults that
 
 ```bash
 # Scan a SINGLE target with a basic port scan and web modules
-bbot -f web-basic -m nmap nuclei --allow-deadly -t app.evilcorp.com​
+bbot -f web-basic -m nmap nuclei --allow-deadly -t app.evilcorp.com
 ```
 
 ```bash
 # Scanning MULTIPLE targets
-bbot -f web-basic -m nmap nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com​
+bbot -f web-basic -m nmap nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
 ```
 
 ```bash
 # Scanning MULTIPLE targets while performing subdomain enumeration
-bbot -f subdomain-enum web-basic -m nmap nuclei –allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com​
+bbot -f subdomain-enum web-basic -m nmap nuclei --allow-deadly -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
 ```
 
 ```bash
-# Scanning MULTIPLE targets on a BUDGET​
-bbot -f subdomain-enum web-basic -m nmap nuclei –allow-deadly –c modules.nuclei.mode=Budget -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com​
+# Scanning MULTIPLE targets on a BUDGET
+bbot -f subdomain-enum web-basic -m nmap nuclei --allow-deadly -c modules.nuclei.mode=budget -t app1.evilcorp.com app2.evilcorp.com app3.evilcorp.com
 ```

docs/scanning/events.md

@@ -87,12 +87,12 @@ BBOT has a sharp distinction between Findings and Vulnerabilities:
 
 **VULNERABILITY**
 
-* There's a higher standard for what is allowed to be a vulnerability. They should be considered **confirmed** and **actionable​** - no additional confirmation required
-* They are always assigned a severity. The possible severities are: LOW, MEDIUM, HIGH, or CRITICAL​
+* There's a higher standard for what is allowed to be a vulnerability. They should be considered **confirmed** and **actionable** - no additional confirmation required
+* They are always assigned a severity. The possible severities are: LOW, MEDIUM, HIGH, or CRITICAL
 
-**FINDING​**
+**FINDING**
 
-* Findings can range anywhere from "slightly interesting behavior" to "likely, but unconfirmed vulnerability"​
+* Findings can range anywhere from "slightly interesting behavior" to "likely, but unconfirmed vulnerability"
 * Are often false positives
 
 By making this separation, actionable vulnerabilities can be identified quickly in the midst of a large scan