Skip to content

Commit

Permalink
merge dev
Browse files Browse the repository at this point in the history
  • Loading branch information
TheTechromancer committed May 3, 2024
2 parents 094db8f + 6fbf389 commit c6cb983
Show file tree
Hide file tree
Showing 15 changed files with 141 additions and 40 deletions.
4 changes: 2 additions & 2 deletions bbot/core/event/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -1223,13 +1223,13 @@ class WAF(DictHostEvent):
class _data_validator(BaseModel):
url: str
host: str
WAF: str
waf: str
info: Optional[str] = None
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)

def _pretty_string(self):
return self.data["WAF"]
return self.data["waf"]


def make_event(
Expand Down
1 change: 1 addition & 0 deletions bbot/defaults.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ url_extension_blacklist:
- woff
- woff2
- ttf
- eot
- sass
- scss
# audio
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/deadly/dastardly.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def construct_command(self, target):
def parse_dastardly_xml(self, xml_file):
try:
with open(xml_file, "rb") as f:
et = etree.parse(f)
et = etree.parse(f, parser=etree.XMLParser(recover=True))
for testsuite in et.iter("testsuite"):
yield TestSuite(testsuite)
except FileNotFoundError:
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/dockerhub.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
class dockerhub(BaseModule):
watched_events = ["SOCIAL", "ORG_STUB"]
produced_events = ["SOCIAL", "CODE_REPOSITORY", "URL_UNVERIFIED"]
flags = ["active", "safe", "code-enum"]
flags = ["passive", "safe", "code-enum"]
meta = {"description": "Search for docker repositories of discovered orgs/usernames"}

site_url = "https://hub.docker.com"
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/internal/excavate.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ async def report(self, result, name, event, **kwargs):


class URLExtractor(BaseExtractor):
url_path_regex = r"((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]+)*/?)"
url_path_regex = r"((?:\w|\d)(?:[\d\w-]+\.?)+(?::\d{1,5})?(?:/[-\w\.\(\)]*[-\w\.]+)*/?)"
regexes = {
"fulluri": r"(?i)" + r"([a-z]\w{1,15})://" + url_path_regex,
"fullurl": r"(?i)" + r"(https?)://" + url_path_regex,
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/output/asset_inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ def absorb_event(self, event):
self.technologies.add(event.data["technology"])

if event.type == "WAF":
if waf := event.data.get("WAF", ""):
if waf := event.data.get("waf", ""):
if update_http_status or not self.waf:
self.waf = waf

Expand Down
38 changes: 21 additions & 17 deletions bbot/modules/social.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,35 +8,39 @@ class social(BaseModule):
meta = {"description": "Look for social media links in webpages"}
flags = ["passive", "safe", "social-enum"]

social_media_regex = {
"linkedin": r"(?:https?://)?(?:www.)?linkedin.com/(?:in|company)/([a-zA-Z0-9-]+)/?",
"facebook": r"(?:https?://)?(?:www.)?facebook.com/([a-zA-Z0-9.]+)/?",
"twitter": r"(?:https?://)?(?:www.)?twitter.com/([a-zA-Z0-9_]{1,15})/?",
"github": r"(?:https?://)?(?:www.)?github.com/([a-zA-Z0-9_-]+)/?",
"instagram": r"(?:https?://)?(?:www.)?instagram.com/([a-zA-Z0-9_.]+)/?",
"youtube": r"(?:https?://)?(?:www.)?youtube.com/@([a-zA-Z0-9_]+)/?",
"bitbucket": r"(?:https?://)?(?:www.)?bitbucket.org/([a-zA-Z0-9_-]+)/?",
"gitlab": r"(?:https?://)?(?:www.)?gitlab.(?:com|org)/([a-zA-Z0-9_-]+)",
"discord": r"(?:https?://)?(?:www.)?discord.gg/([a-zA-Z0-9_-]+)",
"docker": r"(?:https?://)?hub.docker.com/[ru]/([a-zA-Z0-9_-]+)",
"huggingface": r"(?:https?://)?huggingface.co/([a-zA-Z0-9_-]+)",
# platform name : (regex, case_sensitive)
social_media_platforms = {
"linkedin": (r"linkedin.com/(?:in|company)/([a-zA-Z0-9-]+)", False),
"facebook": (r"facebook.com/([a-zA-Z0-9.]+)", False),
"twitter": (r"twitter.com/([a-zA-Z0-9_]{1,15})", False),
"github": (r"github.com/([a-zA-Z0-9_-]+)", False),
"instagram": (r"instagram.com/([a-zA-Z0-9_.]+)", False),
"youtube": (r"youtube.com/@([a-zA-Z0-9_]+)", False),
"bitbucket": (r"bitbucket.org/([a-zA-Z0-9_-]+)", False),
"gitlab": (r"gitlab.(?:com|org)/([a-zA-Z0-9_-]+)", False),
"discord": (r"discord.gg/([a-zA-Z0-9_-]+)", True),
"docker": (r"hub.docker.com/[ru]/([a-zA-Z0-9_-]+)", False),
"huggingface": (r"huggingface.co/([a-zA-Z0-9_-]+)", False),
}

scope_distance_modifier = 1

async def setup(self):
self.compiled_regexes = {k: re.compile(v) for k, v in self.social_media_regex.items()}
self.compiled_regexes = {k: (re.compile(v), c) for k, (v, c) in self.social_media_platforms.items()}
return True

async def handle_event(self, event):
for platform, regex in self.compiled_regexes.items():
for platform, (regex, case_sensitive) in self.compiled_regexes.items():
for match in regex.finditer(event.data):
url = match.group()
if not url.startswith("http"):
url = f"https://{url}"
profile_name = match.groups()[0]
if not case_sensitive:
url = url.lower()
profile_name = profile_name.lower()
social_event = self.make_event(
{"platform": platform, "url": url, "profile_name": profile_name}, "SOCIAL", source=event
{"platform": platform, "url": f"https://{url}", "profile_name": profile_name},
"SOCIAL",
source=event,
)
social_event.scope_distance = event.scope_distance
await self.emit_event(social_event)
4 changes: 2 additions & 2 deletions bbot/modules/wafw00f.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ async def handle_event(self, event):
waf_detections = await self.helpers.run_in_executor(WW.identwaf)
if waf_detections:
for waf in waf_detections:
await self.emit_event({"host": str(event.host), "url": url, "WAF": waf}, "WAF", source=event)
await self.emit_event({"host": str(event.host), "url": url, "waf": waf}, "WAF", source=event)
else:
if self.config.get("generic_detect") == True:
generic = await self.helpers.run_in_executor(WW.genericdetect)
Expand All @@ -47,7 +47,7 @@ async def handle_event(self, event):
{
"host": str(event.host),
"url": url,
"WAF": "generic detection",
"waf": "generic detection",
"info": WW.knowledge["generic"]["reason"],
},
"WAF",
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/zoomeye.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class zoomeye(subdomain_enum_apikey):
"include_related": "Include domains which may be related to the target",
}

base_url = "https://api.zoomeye.org"
base_url = "https://api.zoomeye.hk"

async def setup(self):
self.max_pages = self.config.get("max_pages", 20)
Expand Down
52 changes: 52 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_github_org.py
Original file line number Diff line number Diff line change
Expand Up @@ -392,3 +392,55 @@ def check(self, module_test, events):
and e.scope_distance == 2
]
), "Failed to find TheTechromancer github repo"


class TestGithub_Org_Custom_Target(TestGithub_Org):
targets = ["ORG:blacklanternsecurity"]
config_overrides = {"scope_report_distance": 10, "omit_event_types": [], "speculate": True}

def check(self, module_test, events):
assert len(events) == 7
assert 1 == len(
[e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity" and e.scope_distance == 1]
)
assert 1 == len(
[
e
for e in events
if e.type == "SOCIAL"
and e.data["platform"] == "github"
and e.data["profile_name"] == "blacklanternsecurity"
and e.scope_distance == 1
]
)
assert 1 == len(
[e for e in events if e.type == "DNS_NAME" and e.data == "github.com" and e.scope_distance == 1]
)
assert 1 == len(
[
e
for e in events
if e.type == "URL_UNVERIFIED"
and e.data == "https://github.com/blacklanternsecurity"
and e.scope_distance == 1
]
)
assert 1 == len(
[
e
for e in events
if e.type == "CODE_REPOSITORY"
and e.data["url"] == "https://github.com/blacklanternsecurity/test_keys"
and e.scope_distance == 1
]
)
assert 1 == len(
[
e
for e in events
if e.type == "SOCIAL"
and e.data["platform"] == "github"
and e.data["profile_name"] == "TheTechromancer"
and e.scope_distance == 2
]
)
26 changes: 26 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_httpx.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,29 @@ def check(self, module_test, events):
if e.type.startswith("DNS_NAME") and e.data == "www.evilcorp.com" and "affiliate" in e.tags
]
)


class TestHTTPX_URLBlacklist(ModuleTestBase):
targets = ["http://127.0.0.1:8888"]
modules_overrides = ["httpx", "speculate", "excavate"]
config_overrides = {"web_spider_distance": 10, "web_spider_depth": 10}

async def setup_after_prep(self, module_test):
module_test.httpserver.expect_request("/").respond_with_data(
"""
<a href="/test.aspx"/>
<a href="/test.svg"/>
<a href="/test.woff2"/>
<a href="/test.txt"/>
"""
)

def check(self, module_test, events):
assert 4 == len([e for e in events if e.type == "URL_UNVERIFIED"])
assert 3 == len([e for e in events if e.type == "HTTP_RESPONSE"])
assert 3 == len([e for e in events if e.type == "URL"])
assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/"])
assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/test.aspx"])
assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/test.txt"])
assert not any([e for e in events if "URL" in e.type and ".svg" in e.data])
assert not any([e for e in events if "URL" in e.type and ".woff" in e.data])
33 changes: 25 additions & 8 deletions bbot/test/test_step_2/module_tests/test_module_social.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,38 @@ async def setup_after_prep(self, module_test):
"response_data": """
<html>
<a href="https://discord.gg/asdf"/><a href="https://github.com/blacklanternsecurity/bbot"/>
<a href="https://hub.docker.com/r/blacklanternsecurity"/>
<a href="https://hub.docker.com/r/blacklanternsecurity/bbot"/>
<a href="https://hub.docker.com/r/blacklanternSECURITY/bbot"/>
</html>
"""
}
module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)

def check(self, module_test, events):
assert any(
e.type == "SOCIAL" and e.data["platform"] == "discord" and e.data["profile_name"] == "asdf" for e in events
assert 3 == len([e for e in events if e.type == "SOCIAL"])
assert 1 == len(
[
e
for e in events
if e.type == "SOCIAL" and e.data["platform"] == "discord" and e.data["profile_name"] == "asdf"
]
)
assert any(
e.type == "SOCIAL" and e.data["platform"] == "docker" and e.data["profile_name"] == "blacklanternsecurity"
for e in events
assert 1 == len(
[
e
for e in events
if e.type == "SOCIAL"
and e.data["platform"] == "docker"
and e.data["profile_name"] == "blacklanternsecurity"
]
)
assert any(
e.type == "SOCIAL" and e.data["platform"] == "github" and e.data["profile_name"] == "blacklanternsecurity"
for e in events
assert 1 == len(
[
e
for e in events
if e.type == "SOCIAL"
and e.data["platform"] == "github"
and e.data["profile_name"] == "blacklanternsecurity"
]
)
2 changes: 1 addition & 1 deletion bbot/test/test_step_2/module_tests/test_module_wafw00f.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ async def setup_after_prep(self, module_test):
module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)

def check(self, module_test, events):
assert any(e.type == "WAF" and "LiteSpeed" in e.data["WAF"] for e in events)
assert any(e.type == "WAF" and "LiteSpeed" in e.data["waf"] for e in events)


class TestWafw00f_noredirect(ModuleTestBase):
Expand Down
10 changes: 5 additions & 5 deletions bbot/test/test_step_2/module_tests/test_module_zoomeye.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,24 @@ class TestZoomEye(ModuleTestBase):

async def setup_before_prep(self, module_test):
module_test.httpx_mock.add_response(
url="https://api.zoomeye.org/resources-info",
url="https://api.zoomeye.hk/resources-info",
match_headers={"API-KEY": "asdf"},
json={"quota_info": {"remain_total_quota": 5}},
)
module_test.httpx_mock.add_response(
url="https://api.zoomeye.org/domain/search?q=blacklanternsecurity.com&type=0&page=1",
url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=1",
json={"list": [{"name": "asdf.blacklanternsecurity.com"}]},
)
module_test.httpx_mock.add_response(
url="https://api.zoomeye.org/domain/search?q=blacklanternsecurity.com&type=0&page=2",
url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=2",
json={"list": [{"name": "zzzz.blacklanternsecurity.com"}]},
)
module_test.httpx_mock.add_response(
url="https://api.zoomeye.org/domain/search?q=blacklanternsecurity.com&type=0&page=3",
url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=3",
json={"list": [{"name": "ffff.blacklanternsecurity.com"}, {"name": "affiliate.bls"}]},
)
module_test.httpx_mock.add_response(
url="https://api.zoomeye.org/domain/search?q=blacklanternsecurity.com&type=0&page=4",
url="https://api.zoomeye.hk/domain/search?q=blacklanternsecurity.com&type=0&page=4",
json={"list": [{"name": "nope.blacklanternsecurity.com"}]},
)

Expand Down
1 change: 1 addition & 0 deletions docs/scanning/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ url_extension_blacklist:
- woff
- woff2
- ttf
- eot
- sass
- scss
# audio
Expand Down

0 comments on commit c6cb983

Please sign in to comment.