Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Created module for enumerating AWS S3 Bucket files. #806

Merged
merged 9 commits into from
Nov 20, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
poetry install
- name: Run tests
run: |
poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 600 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
- name: Upload Code Coverage
uses: codecov/codecov-action@v3
with:
Expand Down
16 changes: 7 additions & 9 deletions bbot/core/event/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -867,16 +867,9 @@ def sanitize_data(self, data):

parsed_path_lower = str(self.parsed.path).lower()

url_extension_blacklist = []
url_extension_httpx_only = []
scan = getattr(self, "scan", None)
if scan is not None:
_url_extension_blacklist = scan.config.get("url_extension_blacklist", [])
_url_extension_httpx_only = scan.config.get("url_extension_httpx_only", [])
if _url_extension_blacklist:
url_extension_blacklist = [e.lower() for e in _url_extension_blacklist]
if _url_extension_httpx_only:
url_extension_httpx_only = [e.lower() for e in _url_extension_httpx_only]
url_extension_blacklist = getattr(scan, "url_extension_blacklist", [])
url_extension_httpx_only = getattr(scan, "url_extension_httpx_only", [])

extension = get_file_extension(parsed_path_lower)
if extension:
Expand Down Expand Up @@ -934,6 +927,7 @@ class STORAGE_BUCKET(DictEvent, URL_UNVERIFIED):
class _data_validator(BaseModel):
name: str
url: str
_validate_url = field_validator("url")(validators.validate_url)

def _words(self):
return self.data["name"]
Expand Down Expand Up @@ -1009,6 +1003,7 @@ class _data_validator(BaseModel):
severity: str
description: str
url: Optional[str] = None
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)
_validate_severity = field_validator("severity")(validators.validate_severity)

Expand All @@ -1023,6 +1018,7 @@ class _data_validator(BaseModel):
host: str
description: str
url: Optional[str] = None
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)

def _pretty_string(self):
Expand All @@ -1034,6 +1030,7 @@ class _data_validator(BaseModel):
host: str
technology: str
url: Optional[str] = None
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)

def _data_id(self):
Expand All @@ -1050,6 +1047,7 @@ class _data_validator(BaseModel):
host: str
vhost: str
url: Optional[str] = None
_validate_url = field_validator("url")(validators.validate_url)
_validate_host = field_validator("host")(validators.validate_host)

def _pretty_string(self):
Expand Down
2 changes: 2 additions & 0 deletions bbot/defaults.yml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ url_extension_blacklist:
- woff
- woff2
- ttf
- sass
- scss
# audio
- mp3
- m4a
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/bucket_digitalocean.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@ class bucket_digitalocean(bucket_template):
regions = ["ams3", "fra1", "nyc3", "sfo2", "sfo3", "sgp1"]

def build_url(self, bucket_name, base_domain, region):
return f"https://{bucket_name}.{region}.{base_domain}"
return f"https://{bucket_name}.{region}.{base_domain}/"
48 changes: 48 additions & 0 deletions bbot/modules/bucket_file_enum.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from bbot.modules.base import BaseModule
import xml.etree.ElementTree as ET


class bucket_file_enum(BaseModule):
"""
Enumerate files in a public bucket
"""

watched_events = ["STORAGE_BUCKET"]
produced_events = ["URL_UNVERIFIED"]
meta = {
"description": "Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS"
}
flags = ["passive", "safe", "cloud-enum"]
options = {
"file_limit": 50,
}
options_desc = {"file_limit": "Limit the number of files downloaded per bucket"}
scope_distance_modifier = 2

async def setup(self):
self.file_limit = self.config.get("file_limit", 50)
return True

async def handle_event(self, event):
cloud_tags = (t for t in event.tags if t.startswith("cloud-"))
if any(t.endswith("-amazon") or t.endswith("-digitalocean") for t in cloud_tags):
await self.handle_aws(event)

async def handle_aws(self, event):
url = event.data["url"]
urls_emitted = 0
response = await self.helpers.request(url)
status_code = getattr(response, "status_code", 0)
if status_code == 200:
content = response.text
root = ET.fromstring(content)
namespace = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
keys = [key.text for key in root.findall(".//s3:Key", namespace)]
for key in keys:
bucket_file = url + "/" + key
file_extension = self.helpers.get_file_extension(key)
if file_extension not in self.scan.url_extension_blacklist:
self.emit_event(bucket_file, "URL_UNVERIFIED", source=event, tags="filedownload")
urls_emitted += 1
if urls_emitted >= self.file_limit:
return
13 changes: 9 additions & 4 deletions bbot/modules/filedownload.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ class filedownload(BaseModule):
"max_filesize": "Cancel download if filesize is greater than this size",
}

scope_distance_modifier = 1
scope_distance_modifier = 3

async def setup(self):
self.extensions = list(set([e.lower().strip(".") for e in self.options.get("extensions", [])]))
Expand All @@ -101,8 +101,11 @@ async def filter_event(self, event):
# accept file download requests from other modules
if "filedownload" in event.tags:
return True
if self.hash_event(event) in self.urls_downloaded:
return False, f"Already processed {event}"
else:
if event.scope_distance > 1:
return False, f"{event} not within scope distance"
elif self.hash_event(event) in self.urls_downloaded:
return False, f"Already processed {event}"
return True

def hash_event(self, event):
Expand All @@ -113,7 +116,9 @@ def hash_event(self, event):
async def handle_event(self, event):
if event.type == "URL_UNVERIFIED":
url_lower = event.data.lower()
if any(url_lower.endswith(f".{e}") for e in self.extensions):
extension_matches = any(url_lower.endswith(f".{e}") for e in self.extensions)
filedownload_requested = "filedownload" in event.tags
if extension_matches or filedownload_requested:
await self.download_file(event.data)
elif event.type == "HTTP_RESPONSE":
content_type = event.data["header"].get("content_type", "")
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/templates/bucket.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def valid_bucket_name(self, bucket_name):
return False

def build_url(self, bucket_name, base_domain, region):
return f"https://{bucket_name}.{base_domain}"
return f"https://{bucket_name}.{base_domain}/"

def gen_tags_exists(self, response):
return set()
Expand Down
4 changes: 4 additions & 0 deletions bbot/scanner/scanner.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,10 @@ def __init__(
)
self.scope_report_distance = int(self.config.get("scope_report_distance", 1))

# url file extensions
self.url_extension_blacklist = set(e.lower() for e in self.config.get("url_extension_blacklist", []))
self.url_extension_httpx_only = set(e.lower() for e in self.config.get("url_extension_httpx_only", []))

# custom HTTP headers warning
self.custom_http_headers = self.config.get("http_headers", {})
if self.custom_http_headers:
Expand Down
2 changes: 1 addition & 1 deletion bbot/test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def pytest_sessionfinish(session, exitstatus):

@pytest.fixture
def non_mocked_hosts() -> list:
return ["127.0.0.1", "localhost", "githubusercontent.com"] + interactsh_servers
return ["127.0.0.1", "localhost", "raw.githubusercontent.com"] + interactsh_servers


@pytest.fixture
Expand Down
3 changes: 0 additions & 3 deletions bbot/test/test_step_1/test_cloud_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ async def test_cloud_helpers(bbot_scanner, bbot_config):
for provider_name in provider_names:
assert provider_name in scan1.helpers.cloud.providers.providers

log.critical(scan1.helpers.cloud.providers.providers)
for p in scan1.helpers.cloud.providers.providers.values():
print(f"{p.name}: {p.domains} / {p.ranges}")
amazon_ranges = list(scan1.helpers.cloud["amazon"].ranges)
Expand All @@ -30,12 +29,10 @@ async def test_cloud_helpers(bbot_scanner, bbot_config):
other_event3._resolved_hosts = {"asdf.amazonaws.com"}

for event in (ip_event, aws_event1, aws_event2, aws_event4, other_event2, other_event3):
log.critical(event)
await scan1.helpers.cloud.tag_event(event)
assert "cloud-amazon" in event.tags, f"{event} was not properly cloud-tagged"

for event in (aws_event3, other_event1):
log.critical(event)
await scan1.helpers.cloud.tag_event(event)
assert "cloud-amazon" not in event.tags, f"{event} was improperly cloud-tagged"
assert not any(
Expand Down
10 changes: 5 additions & 5 deletions bbot/test/test_step_2/module_tests/test_module_bucket_amazon.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ def modules_overrides(self):
return ["excavate", "speculate", "httpx", self.module_name]

def url_setup(self):
self.url_1 = f"https://{self.random_bucket_1}"
self.url_2 = f"https://{self.random_bucket_2}"
self.url_3 = f"https://{self.random_bucket_3}"
self.url_1 = f"https://{self.random_bucket_1}/"
self.url_2 = f"https://{self.random_bucket_2}/"
self.url_3 = f"https://{self.random_bucket_3}/"

def bucket_setup(self):
self.url_setup()
Expand Down Expand Up @@ -83,14 +83,14 @@ def check(self, module_test, events):
url = e.data.get("url", "")
assert self.random_bucket_2 in url
assert not self.random_bucket_1 in url
assert not f"{self.random_bucket_3}" in url
assert not self.random_bucket_3 in url
# make sure bucket mutations were found
assert any(
e.type == "STORAGE_BUCKET"
and str(e.module) == self.module_name
and f"{random_bucket_name_3}" in e.data["url"]
for e in events
), f'bucket (dev mutation) not found for module "{self.module_name}"'
), f'bucket (dev mutation: {self.random_bucket_3}) not found for module "{self.module_name}"'


class TestBucket_Amazon(Bucket_Amazon_Base):
Expand Down
40 changes: 40 additions & 0 deletions bbot/test/test_step_2/module_tests/test_module_bucket_file_enum.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from .base import ModuleTestBase


class TestBucket_File_Enum(ModuleTestBase):
targets = ["http://127.0.0.1:8888"]
modules_overrides = ["bucket_file_enum", "filedownload", "httpx", "excavate"]
config_overrides = {"scope_report_distance": 5}

open_bucket_url = "https://testbucket.s3.amazonaws.com/"
open_bucket_body = """<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>testbucket</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>index.html</Key><LastModified>2023-05-22T23:04:38.000Z</LastModified><ETag>&quot;4a2d2d114f3abf90f8bd127c1f25095a&quot;</ETag><Size>5</Size><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>test.pdf</Key><LastModified>2022-04-30T21:13:40.000Z</LastModified><ETag>&quot;723b0018c2f5a7ef06a34f84f6fa97e4&quot;</ETag><Size>388901</Size><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>"""

pdf_data = """%PDF-1.
1 0 obj<</Pages 2 0 R>>endobj
2 0 obj<</Kids[3 0 R]/Count 1>>endobj
3 0 obj<</Parent 2 0 R>>endobj
trailer <</Root 1 0 R>>"""

async def setup_before_prep(self, module_test):
module_test.httpserver.expect_request("/").respond_with_data(f'<a href="{self.open_bucket_url}"/>')
module_test.httpx_mock.add_response(
url=self.open_bucket_url,
text=self.open_bucket_body,
)
module_test.httpx_mock.add_response(
url=f"{self.open_bucket_url}test.pdf",
text=self.pdf_data,
headers={"Content-Type": "application/pdf"},
)
module_test.httpx_mock.add_response(
url=f"{self.open_bucket_url}test.css",
text="",
)

def check(self, module_test, events):
download_dir = module_test.scan.home / "filedownload"
files = list(download_dir.glob("*.pdf"))
assert any(e.type == "URL_UNVERIFIED" and e.data.endswith("test.pdf") for e in events)
assert not any(e.type == "URL_UNVERIFIED" and e.data.endswith("test.css") for e in events)
assert any(f.name.endswith("test.pdf") for f in files), "Failed to download PDF file from open bucket"
assert not any(f.name.endswith("test.css") for f in files), "Unwanted CSS file was downloaded"
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,6 @@ class TestFileDownload(ModuleTestBase):
3 0 obj<</Parent 2 0 R>>endobj
trailer <</Root 1 0 R>>"""

async def setup_before_prep(self, module_test):
module_test.httpx_mock.add_response(
url="https://raw.githubusercontent.com/jshttp/mime-db/master/db.json",
json={
"application/pdf": {"source": "iana", "compressible": False, "extensions": ["pdf"]},
},
)

async def setup_after_prep(self, module_test):
module_test.set_expect_requests(
dict(uri="/"),
Expand Down
6 changes: 0 additions & 6 deletions bbot/test/test_step_2/module_tests/test_module_massdns.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,6 @@ class TestMassdns(ModuleTestBase):
subdomain_wordlist = tempwordlist(["www", "asdf"])
config_overrides = {"modules": {"massdns": {"wordlist": str(subdomain_wordlist)}}}

async def setup_before_prep(self, module_test):
module_test.httpx_mock.add_response(
url="https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt",
text="8.8.8.8\n8.8.4.4\n1.1.1.1",
)

def check(self, module_test, events):
assert any(e.data == "www.blacklanternsecurity.com" for e in events)
assert not any(e.data == "asdf.blacklanternsecurity.com" for e in events)
8 changes: 0 additions & 8 deletions bbot/test/test_step_2/module_tests/test_module_secretsdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,6 @@ class TestSecretsDB(ModuleTestBase):
modules_overrides = ["httpx", "secretsdb"]

async def setup_before_prep(self, module_test):
module_test.httpx_mock.add_response(
url="https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml",
text="""patterns:
- pattern:
confidence: 99
name: Asymmetric Private Key
regex: '-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'""",
)
expect_args = {"method": "GET", "uri": "/"}
respond_args = {"response_data": "-----BEGIN PGP PRIVATE KEY BLOCK-----"}
module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
Expand Down
19 changes: 0 additions & 19 deletions bbot/test/test_step_2/module_tests/test_module_subdomain_hijack.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,6 @@ class TestSubdomain_Hijack(ModuleTestBase):
targets = ["http://127.0.0.1:8888"]
modules_overrides = ["httpx", "excavate", "subdomain_hijack"]

async def setup_before_prep(self, module_test):
module_test.httpx_mock.add_response(
url="https://raw.githubusercontent.com/EdOverflow/can-i-take-over-xyz/master/fingerprints.json",
json=[
{
"cicd_pass": True,
"cname": ["us-east-1.elasticbeanstalk.com"],
"discussion": "[Issue #194](https://github.com/EdOverflow/can-i-take-over-xyz/issues/194)",
"documentation": "",
"fingerprint": "NXDOMAIN",
"http_status": None,
"nxdomain": True,
"service": "AWS/Elastic Beanstalk",
"status": "Vulnerable",
"vulnerable": True,
}
],
)

async def setup_after_prep(self, module_test):
fingerprints = module_test.module.fingerprints
assert fingerprints, "No subdomain hijacking fingerprints available"
Expand Down
17 changes: 3 additions & 14 deletions bbot/test/test_step_2/module_tests/test_module_web_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,14 @@ class TestWebReport(ModuleTestBase):

async def setup_before_prep(self, module_test):
# secretsdb --> FINDING
module_test.httpx_mock.add_response(
url="https://raw.githubusercontent.com/blacklanternsecurity/secrets-patterns-db/master/db/rules-stable.yml",
text="""patterns:
- pattern:
confidence: 99
name: Asymmetric Private Key
regex: '-----BEGIN ((EC|PGP|DSA|RSA|OPENSSH) )?PRIVATE KEY( BLOCK)?-----'""",
)
# wappalyzer --> TECHNOLOGY
# badsecrets --> VULNERABILITY
respond_args = {"response_data": web_body}
module_test.set_expect_requests(respond_args=respond_args)

def check(self, module_test, events):
for e in events:
module_test.log.critical(e)
report_file = module_test.scan.home / "web_report.html"
with open(report_file) as f:
report_content = f.read()
Expand All @@ -31,12 +25,7 @@ def check(self, module_test, events):
<li><strong>http://127.0.0.1:8888/</strong>"""
in report_content
)
assert (
"""<h3>FINDING</h3>
<ul>
<li>Possible secret (Asymmetric Private Key)"""
in report_content
)
assert """Possible secret (Asymmetric Private Key)""" in report_content
assert "<h3>TECHNOLOGY</h3>" in report_content
assert "<p>flask</p>" in report_content

Expand Down