Merge branch 'dev' into bypass403-diff-bug

dependabot.yml → .github/dependabot.yml

@@ -4,4 +4,5 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     open-pull-requests-limit: 10

README.md

@@ -263,7 +263,7 @@ For a full list of modules, including the data types consumed and emitted by eac
 | passive          | 57          | Never connects to target systems              | affiliates, aggregate, anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crobat, crt, dehashed, digitorus, dnscommonsrv, dnsdumpster, emailformat, excavate, fullhunt, github_codesearch, github_org, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, massdns, myssl, nsec, otx, passivetotal, pgp, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, sublist3r, threatminer, urlscan, viewdns, virustotal, wayback, zoomeye                                                                                                                                                                                                                               |
 | subdomain-enum   | 47          | Enumerates subdomains                         | anubisdb, asn, azure_realm, azure_tenant, bevigil, binaryedge, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnscommonsrv, dnsdumpster, dnszonetransfer, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, massdns, myssl, nsec, oauth, otx, passivetotal, postman, rapiddns, riddler, securitytrails, shodan_dns, sitedossier, sslcert, subdomain_hijack, subdomaincenter, subdomains, threatminer, urlscan, virustotal, wayback, zoomeye                                                                                                                                                                                                                                                                                                                               |
 | active           | 39          | Makes active connections to target systems    | ajaxpro, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dnszonetransfer, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gowitness, host_header, httpx, hunt, iis_shortnames, masscan, nmap, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, vhost, wafw00f, wappalyzer                                                                                                                                                                                                                                                                                                                                                               |
-| web-thorough     | 26          | More advanced web scanning functionality      | ajaxpro, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, ffuf_shortnames, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| web-thorough     | 29          | More advanced web scanning functionality      | ajaxpro, azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, ffuf_shortnames, filedownload, generic_ssrf, git, host_header, httpx, hunt, iis_shortnames, nmap, ntlm, oauth, robots, secretsdb, smuggler, sslcert, subdomain_hijack, telerik, url_manipulation, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
 | aggressive       | 19          | Generates a large amount of network traffic   | bypass403, dastardly, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, masscan, massdns, nmap, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | web-basic        | 17          | Basic, non-intrusive web scan functionality   | azure_realm, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, sslcert, subdomain_hijack, wappalyzer                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | cloud-enum       | 11          | Enumerates cloud resources                    | azure_realm, azure_tenant, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth, subdomain_hijack                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |

bbot/core/event/base.py

@@ -1,3 +1,4 @@
+import re
 import json
 import asyncio
 import logging
@@ -6,25 +7,28 @@
 from typing import Optional
 from datetime import datetime
 from contextlib import suppress
+from urllib.parse import urljoin
 from pydantic import BaseModel, field_validator
 
 from .helpers import *
 from bbot.core.errors import *
 from bbot.core.helpers import (
     extract_words,
-    split_host_port,
+    get_file_extension,
     host_in_host,
     is_domain,
     is_subdomain,
     is_ip,
     is_ptr,
+    is_uri,
     domain_stem,
     make_netloc,
     make_ip_type,
+    recursive_decode,
     smart_decode,
-    get_file_extension,
-    validators,
+    split_host_port,
     tagify,
+    validators,
 )
 
 
@@ -866,6 +870,8 @@ def _words(self):
 
 
 class URL_UNVERIFIED(BaseEvent):
+    _status_code_regex = re.compile(r"^status-(\d{1,3})$")
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.web_spider_distance = getattr(self.source, "web_spider_distance", 0)
@@ -921,6 +927,14 @@ def _data_id(self):
             data = "spider-danger" + data
         return data
 
+    @property
+    def http_status(self):
+        for t in self.tags:
+            match = self._status_code_regex.match(t)
+            if match:
+                return int(match.groups()[0])
+        return 0
+
 
 class URL(URL_UNVERIFIED):
     def sanitize_data(self, data):
@@ -973,7 +987,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         # count number of consecutive redirects
         self.num_redirects = getattr(self.source, "num_redirects", 0)
-        if str(self.data.get("status_code", 0)).startswith("3"):
+        if str(self.http_status).startswith("3"):
             self.num_redirects += 1
 
     def sanitize_data(self, data):
@@ -1001,6 +1015,34 @@ def _words(self):
     def _pretty_string(self):
         return f'{self.data["hash"]["header_mmh3"]}:{self.data["hash"]["body_mmh3"]}'
 
+    @property
+    def http_status(self):
+        try:
+            return int(self.data.get("status_code", 0))
+        except (ValueError, TypeError):
+            return 0
+
+    @property
+    def http_title(self):
+        http_title = self.data.get("title", "")
+        try:
+            return recursive_decode(http_title)
+        except Exception:
+            return http_title
+
+    @property
+    def redirect_location(self):
+        location = self.data.get("location", "")
+        # if it's a redirect
+        if location:
+            # get the url scheme
+            scheme = is_uri(location, return_scheme=True)
+            # if there's no scheme (i.e. it's a relative redirect)
+            if not scheme:
+                # then join the location with the current url
+                location = urljoin(self.parsed.geturl(), location)
+        return location
+
 
 class VULNERABILITY(DictHostEvent):
     _always_emit = True
@@ -1123,6 +1165,7 @@ class SOCIAL(DictEvent):
 
 class WEBSCREENSHOT(DictHostEvent):
     _always_emit = True
+    _quick_emit = True
 
 
 class AZURE_TENANT(DictEvent):
@@ -1203,10 +1246,11 @@ def make_event(
     """
 
     # allow tags to be either a string or an array
-    if tags is not None:
-        if isinstance(tags, str):
-            tags = [tags]
-        tags = list(tags)
+    if not tags:
+        tags = []
+    elif isinstance(tags, str):
+        tags = [tags]
+    tags = list(tags)
 
     if is_event(data):
         if scan is not None and not data.scan:

bbot/core/helpers/async_helpers.py

@@ -57,33 +57,34 @@ def __init__(self):
 
     @property
     def value(self):
-        return len(self.tasks)
+        return sum([t.n for t in self.tasks.values()])
 
-    def count(self, task_name, _log=True):
+    def count(self, task_name, n=1, _log=True):
         if callable(task_name):
             task_name = f"{task_name.__qualname__}()"
-        return self.Task(self, task_name, _log)
+        return self.Task(self, task_name, n=n, _log=_log)
 
     class Task:
-        def __init__(self, manager, task_name, _log=True):
+        def __init__(self, manager, task_name, n=1, _log=True):
             self.manager = manager
             self.task_name = task_name
             self.task_id = None
             self.start_time = None
             self.log = _log
+            self.n = n
 
         async def __aenter__(self):
-            self.task_id = uuid.uuid4()  # generate a unique ID for the task
+            self.task_id = uuid.uuid4()
             # if self.log:
             #     log.trace(f"Starting task {self.task_name} ({self.task_id})")
-            async with self.manager.lock:  # acquire the lock
+            async with self.manager.lock:
                 self.start_time = datetime.now()
                 self.manager.tasks[self.task_id] = self
-            return self.task_id  # this will be passed as 'task_id' to __aexit__
+            return self
 
         async def __aexit__(self, exc_type, exc_val, exc_tb):
-            async with self.manager.lock:  # acquire the lock
-                self.manager.tasks.pop(self.task_id, None)  # remove only current task
+            async with self.manager.lock:
+                self.manager.tasks.pop(self.task_id, None)
             # if self.log:
             #     log.trace(f"Finished task {self.task_name} ({self.task_id})")
 

bbot/core/helpers/helper.py

@@ -77,8 +77,8 @@ def __init__(self, config, scan=None):
         # cloud helpers
         self.cloud = CloudHelper(self)
 
-    def interactsh(self):
-        return Interactsh(self)
+    def interactsh(self, *args, **kwargs):
+        return Interactsh(self, *args, **kwargs)
 
     def http_compare(self, url, allow_redirects=False, include_cache_buster=True):
         return HttpCompare(url, self, allow_redirects=allow_redirects, include_cache_buster=include_cache_buster)

bbot/core/helpers/interactsh.py

@@ -78,12 +78,13 @@ class Interactsh:
         ```
     """
 
-    def __init__(self, parent_helper):
+    def __init__(self, parent_helper, poll_interval=10):
         self.parent_helper = parent_helper
         self.server = None
         self.correlation_id = None
         self.custom_server = self.parent_helper.config.get("interactsh_server", None)
         self.token = self.parent_helper.config.get("interactsh_token", None)
+        self.poll_interval = poll_interval
         self._poll_task = None
 
     async def register(self, callback=None):
@@ -234,6 +235,8 @@ async def poll(self):
             r = await self.parent_helper.request(
                 f"https://{self.server}/poll?id={self.correlation_id}&secret={self.secret}", headers=headers
             )
+            if r is None:
+                raise InteractshError("Error polling interact.sh: No response from server")
 
             ret = []
             data_list = r.json().get("data", None)
@@ -279,7 +282,7 @@ async def _poll_loop(self, callback):
                 log.warning(e)
                 log.trace(traceback.format_exc())
             if not data_list:
-                await asyncio.sleep(10)
+                await asyncio.sleep(self.poll_interval)
                 continue
             for data in data_list:
                 if data:

bbot/core/helpers/misc.py

@@ -389,6 +389,50 @@ def url_parents(u):
             u = parent
 
 
+def best_http_status(code1, code2):
+    """
+    Determine the better HTTP status code between two given codes.
+
+    The 'better' status code is considered based on typical usage and priority in HTTP communication.
+    Lower codes are generally better than higher codes. Within the same class (e.g., 2xx), a lower code is better.
+    Between different classes, the order of preference is 2xx > 3xx > 1xx > 4xx > 5xx.
+
+    Args:
+        code1 (int): The first HTTP status code.
+        code2 (int): The second HTTP status code.
+
+    Returns:
+        int: The better HTTP status code between the two provided codes.
+
+    Examples:
+        >>> better_http_status(200, 404)
+        200
+        >>> better_http_status(500, 400)
+        400
+        >>> better_http_status(301, 302)
+        301
+    """
+
+    # Classify the codes into their respective categories (1xx, 2xx, 3xx, 4xx, 5xx)
+    def classify_code(code):
+        return int(code) // 100
+
+    class1 = classify_code(code1)
+    class2 = classify_code(code2)
+
+    # Priority order for classes
+    priority_order = {2: 1, 3: 2, 1: 3, 4: 4, 5: 5}
+
+    # Compare based on class priority
+    p1 = priority_order.get(class1, 10)
+    p2 = priority_order.get(class2, 10)
+    if p1 != p2:
+        return code1 if p1 < p2 else code2
+
+    # If in the same class, the lower code is better
+    return min(code1, code2)
+
+
 def tldextract(data):
     """
     Extracts the subdomain, domain, and suffix from a URL string.
@@ -1121,6 +1165,8 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True):
     This function takes a list `l` and flattens it by splitting its entries on commas.
     It also allows you to optionally open entries as files and add their contents to the list.
 
+    The order of entries is preserved, and deduplication is performed automatically.
+
     Args:
         l (list): The list of strings to chain together.
         try_files (bool, optional): Whether to try to open entries as files. Defaults to False.
@@ -1137,6 +1183,8 @@ def chain_lists(l, try_files=False, msg=None, remove_blank=True):
         >>> chain_lists(["a,file.txt", "c,d"], try_files=True)
         ['a', 'f_line1', 'f_line2', 'f_line3', 'c', 'd']
     """
+    if isinstance(l, str):
+        l = [l]
     final_list = dict()
     for entry in l:
         for s in entry.split(","):

bbot/core/helpers/names_generator.py

@@ -298,6 +298,7 @@
     "ashley",
     "audrey",
     "austin",
+    "azathoth",
     "baggins",
     "bailey",
     "barbara",
@@ -347,8 +348,10 @@
     "courtney",
     "craig",
     "crystal",
+    "cthulu",
     "curtis",
     "cynthia",
+    "dagon",
     "dale",
     "dandelion",
     "daniel",
@@ -554,6 +557,7 @@
     "noah",
     "norma",
     "norman",
+    "nyarlathotep",
     "obama",
     "olivia",
     "padme",

bbot/modules/ajaxpro.py

@@ -24,7 +24,7 @@ async def handle_event(self, event):
                     probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx")
                     if probe_confirm:
                         if probe_confirm.status_code != 200:
-                            self.emit_event(
+                            await self.emit_event(
                                 {
                                     "host": str(event.host),
                                     "url": event.data,
@@ -40,7 +40,7 @@ async def handle_event(self, event):
                 ajaxpro_regex_result = self.ajaxpro_regex.search(resp_body)
                 if ajaxpro_regex_result:
                     ajax_pro_path = ajaxpro_regex_result.group(0)
-                    self.emit_event(
+                    await self.emit_event(
                         {
                             "host": str(event.host),
                             "url": event.data["url"],

bbot/modules/azure_realm.py

@@ -4,7 +4,7 @@
 class azure_realm(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["URL_UNVERIFIED"]
-    flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "passive", "safe"]
+    flags = ["affiliates", "subdomain-enum", "cloud-enum", "web-basic", "web-thorough", "passive", "safe"]
     meta = {"description": 'Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm'}
 
     async def setup(self):
@@ -22,7 +22,7 @@ async def handle_event(self, event):
                     auth_url, "URL_UNVERIFIED", source=event, tags=["affiliate", "ms-auth-url"]
                 )
                 url_event.source_domain = domain
-                self.emit_event(url_event)
+                await self.emit_event(url_event)
 
     async def getuserrealm(self, domain):
         url = f"https://login.microsoftonline.com/getuserrealm.srf?login=test@{domain}"