Merge branch 'dev' into bbot-2.0

bbot/core/event/base.py

@@ -968,6 +968,11 @@ class _data_validator(BaseModel):
         url: str
         _validate_url = field_validator("url")(validators.validate_url)
 
+    def sanitize_data(self, data):
+        data = super().sanitize_data(data)
+        data["name"] = data["name"].lower()
+        return data
+
     def _words(self):
         return self.data["name"]
 

bbot/core/helpers/misc.py

@@ -25,6 +25,7 @@
 from tabulate import tabulate
 import wordninja as _wordninja
 from contextlib import suppress
+from unidecode import unidecode  # noqa F401
 import cloudcheck as _cloudcheck
 import tldextract as _tldextract
 import xml.etree.ElementTree as ET

bbot/modules/deadly/dastardly.py

@@ -107,7 +107,9 @@ def parse_dastardly_xml(self, xml_file):
                 et = etree.parse(f)
                 for testsuite in et.iter("testsuite"):
                     yield TestSuite(testsuite)
-        except Exception as e:
+        except FileNotFoundError:
+            pass
+        except etree.ParseError as e:
             self.warning(f"Error parsing Dastardly XML at {xml_file}: {e}")
 
 

bbot/modules/github_org.py

@@ -76,6 +76,8 @@ async def handle_event(self, event):
             user = event.data
             self.verbose(f"Validating whether the organization {user} is within our scope...")
             is_org, in_scope = await self.validate_org(user)
+            if "target" in event.tags:
+                in_scope = True
             if not is_org or not in_scope:
                 self.verbose(f"Unable to validate that {user} is in-scope, skipping...")
                 return

bbot/modules/internal/speculate.py

@@ -153,7 +153,9 @@ async def handle_event(self, event):
             if registered_domain:
                 tld_stub = getattr(tldextracted, "domain", "")
                 if tld_stub:
-                    org_stubs.add(tld_stub)
+                    decoded_tld_stub = self.helpers.smart_decode_punycode(tld_stub)
+                    org_stubs.add(decoded_tld_stub)
+                    org_stubs.add(self.helpers.unidecode(decoded_tld_stub))
         elif event.type == "SOCIAL":
             stub = event.data.get("stub", "")
             if stub:

bbot/modules/templates/bucket.py

@@ -45,12 +45,13 @@ async def handle_event(self, event):
 
     async def handle_dns_name(self, event):
         buckets = set()
-        base = event.data
+        base = self.helpers.unidecode(self.helpers.smart_decode_punycode(event.data))
         stem = self.helpers.domain_stem(base)
         for b in [base, stem]:
             split = b.split(".")
             for d in self.delimiters:
-                buckets.add(d.join(split))
+                bucket_name = d.join(split)
+                buckets.add(bucket_name)
         async for bucket_name, url, tags in self.brute_buckets(buckets, permutations=self.permutations):
             await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags)
 

bbot/scanner/target.py

@@ -1,3 +1,4 @@
+import re
 import logging
 import ipaddress
 from contextlib import suppress
@@ -86,6 +87,10 @@ def __init__(self, scan, *targets, strict_scope=False, make_in_scope=False):
         self.scan = scan
         self.strict_scope = strict_scope
         self.make_in_scope = make_in_scope
+        self.special_event_types = {
+            "ORG_STUB": re.compile(r"^ORG:(.*)", re.IGNORECASE),
+            "ASN": re.compile(r"^ASN:(.*)", re.IGNORECASE),
+        }
 
         self._dummy_module = TargetDummyModule(scan)
         self._events = dict()
@@ -124,6 +129,12 @@ def add_target(self, t, event_type=None):
             if is_event(t):
                 event = t
             else:
+                for eventtype, regex in self.special_event_types.items():
+                    match = regex.match(t)
+                    if match:
+                        t = match.groups()[0]
+                        event_type = eventtype
+                        break
                 try:
                     event = self.scan.make_event(
                         t,

bbot/test/test_step_1/test_events.py

@@ -444,3 +444,12 @@ async def test_events(events, scan, helpers, bbot_config):
     event_5 = scan.make_event("127.0.0.5", source=event_4)
     assert event_5.get_sources() == [event_4, event_3, event_2, event_1, scan.root_event]
     assert event_5.get_sources(omit=True) == [event_4, event_2, event_1, scan.root_event]
+
+    # test storage bucket validation
+    bucket_event = scan.make_event(
+        {"name": "ASDF.s3.amazonaws.com", "url": "https://ASDF.s3.amazonaws.com"},
+        "STORAGE_BUCKET",
+        source=scan.root_event,
+    )
+    assert bucket_event.data["name"] == "asdf.s3.amazonaws.com"
+    assert bucket_event.data["url"] == "https://asdf.s3.amazonaws.com/"

pyproject.toml

@@ -30,7 +30,7 @@ omegaconf = "^2.3.0"
 psutil = "^5.9.4"
 wordninja = "^2.0.0"
 ansible-runner = "^2.3.2"
-deepdiff = "^6.2.3"
+deepdiff = ">=6.2.3,<8.0.0"
 xmltojson = "^2.0.2"
 pycryptodome = "^3.17"
 idna = "^3.4"
@@ -47,6 +47,7 @@ cloudcheck = ">=2.1.0.181,<4.0.0.0"
 tldextract = "^5.1.1"
 cachetools = "^5.3.2"
 socksio = "^1.0.0"
+unidecode = "^1.3.8"
 
 [tool.poetry.group.dev.dependencies]
 flake8 = ">=6,<8"