diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 7b2bd51dd4..95f9d7b5f0 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -30,7 +30,7 @@ jobs: elif [ "$ID" = "arch" ]; then pacman -Syu --noconfirm curl git bash base-devel elif [ "$ID" = "fedora" ]; then - dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel + dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5 elif [ "$ID" = "gentoo" ]; then echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd emerge-webrsync diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 80ded37a6f..944de463ba 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -248,7 +248,7 @@ def tasks(self, module, tasks): return success def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): - _ansible_args = {"ansible_connection": "local"} + _ansible_args = {"ansible_connection": "local", "ansible_python_interpreter": sys.executable} if ansible_args is not None: _ansible_args.update(ansible_args) module_args = None diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index afb89b1492..0664f954a8 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -1,6 +1,5 @@ import random import ipaddress -from pathlib import Path from bbot.core.helpers import validators from bbot.modules.internal.base import BaseInternalModule @@ -24,9 +23,8 @@ class speculate(BaseInternalModule): "SOCIAL", "AZURE_TENANT", "USERNAME", - "FILESYSTEM", ] - produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB", "FILESYSTEM"] + produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"] flags = ["passive"] meta = { "description": "Derive certain event types from others by common sense", @@ -34,11 +32,10 @@ class speculate(BaseInternalModule): "author": "@liquidsec", } - options = {"max_hosts": 65536, "ports": "80,443", "ignore_folders": [".git"]} + options = {"max_hosts": 65536, "ports": "80,443"} options_desc = { "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events", "ports": "The set of ports to speculate on", - "ignore_folders": "Subfolders to ignore when crawling downloaded folders", } scope_distance_modifier = 1 _priority = 4 @@ -75,13 +72,6 @@ async def setup(self): self.hugewarning(f'Enabling the "portscan" module is highly recommended') self.range_to_ip = False - self.ignored_folders = self.config.get("ignore_folders", []) - - return True - - async def filter_event(self, event): - if event.type == "FILESYSTEM" and "folder" not in event.tags: - return False, "Event is not a folder" return True async def handle_event(self, event): @@ -205,14 +195,3 @@ async def handle_event(self, event): email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=["affiliate"]) if email_event: await self.emit_event(email_event, context="detected {event.type}: {event.data}") - - # FILESYSTEM (folder) --> FILESYSTEM (files) - if event.type == "FILESYSTEM": - folder_path = Path(event.data["path"]) - for file_path in folder_path.rglob("*"): - # If the file is not in an ignored folder and if it has an allowed extension raise it as a FILESYSTEM event - if not any(ignored_folder in str(file_path) for ignored_folder in self.ignored_folders): - file_event = self.make_event( - {"path": str(file_path)}, "FILESYSTEM", tags=["parsed_folder", "file"], parent=event - ) - await self.emit_event(file_event) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 148d621003..7435f43c40 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -13,7 +13,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.83.1", + "version": "3.83.3", "config": "", "only_verified": True, "concurrency": 8, diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index a8fe471b4b..386dbbe182 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -376,6 +376,9 @@ def merge(self, other): # misc self.force_start = self.force_start | other.force_start self._cli = self._cli | other._cli + # transfer args + if other._args is not None: + self._args = other._args def bake(self, scan=None): """ diff --git a/bbot/test/test_step_2/module_tests/test_module_git_clone.py b/bbot/test/test_step_2/module_tests/test_module_git_clone.py index f0b91a2a15..15bc54fb37 100644 --- a/bbot/test/test_step_2/module_tests/test_module_git_clone.py +++ b/bbot/test/test_step_2/module_tests/test_module_git_clone.py @@ -215,7 +215,7 @@ class TestGit_CloneWithBlob(TestGit_Clone): config_overrides = {"folder_blobs": True} def check(self, module_test, events): - filesystem_events = [e for e in events if e.type == "FILESYSTEM" and "folder" in e.tags] + filesystem_events = [e for e in events if e.type == "FILESYSTEM"] assert len(filesystem_events) == 1 assert all(["blob" in e.data for e in filesystem_events]) filesystem_event = filesystem_events[0] diff --git a/docs/data/chord_graph/entities.json b/docs/data/chord_graph/entities.json index 783e0f4330..2d8baf1489 100644 --- a/docs/data/chord_graph/entities.json +++ b/docs/data/chord_graph/entities.json @@ -197,7 +197,6 @@ "name": "FILESYSTEM", "parent": 88888888, "consumes": [ - 123, 131, 132 ], @@ -207,8 +206,7 @@ 72, 78, 81, - 112, - 123 + 112 ] }, { @@ -1730,7 +1728,6 @@ 124, 7, 22, - 10, 2, 12, 110, @@ -1742,7 +1739,6 @@ ], "produces": [ 7, - 10, 4, 12, 15, diff --git a/docs/data/chord_graph/rels.json b/docs/data/chord_graph/rels.json index afbe6896dc..78ab196e1f 100644 --- a/docs/data/chord_graph/rels.json +++ b/docs/data/chord_graph/rels.json @@ -1344,11 +1344,6 @@ "target": 22, "type": "consumes" }, - { - "source": 123, - "target": 10, - "type": "consumes" - }, { "source": 123, "target": 2, @@ -1394,11 +1389,6 @@ "target": 123, "type": "produces" }, - { - "source": 10, - "target": 123, - "type": "produces" - }, { "source": 4, "target": 123, diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index e19bb7719f..54249c22eb 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -1,137 +1,137 @@ # List of Modules -| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | Author | Created Date | -|----------------------|----------|-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------|------------------|----------------| -| ajaxpro | scan | No | Check for potentially vulnerable Ajaxpro instances | active, safe, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2024-01-18 | -| baddns | scan | No | Check hosts for domain/subdomain takeovers | active, baddns, cloud-enum, safe, subdomain-hijack, web-basic | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING, VULNERABILITY | @liquidsec | 2024-01-18 | -| baddns_direct | scan | No | Check for unusual subdomain / service takeover edge cases that require direct detection | active, baddns, cloud-enum, safe, subdomain-enum | STORAGE_BUCKET, URL | FINDING, VULNERABILITY | @liquidsec | 2024-01-29 | -| baddns_zone | scan | No | Check hosts for DNS zone transfers and NSEC walks | active, baddns, cloud-enum, safe, subdomain-enum | DNS_NAME | FINDING, VULNERABILITY | @liquidsec | 2024-01-29 | -| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic | HTTP_RESPONSE | FINDING, TECHNOLOGY, VULNERABILITY | @liquidsec | 2022-11-19 | -| bucket_amazon | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | -| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | -| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-08 | -| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2023-03-20 | -| bucket_google | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | -| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-07-05 | -| dastardly | scan | No | Lightweight web application security scanner | active, aggressive, deadly, slow, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | @domwhewell-sage | 2023-12-11 | -| dnsbrute | scan | No | Brute-force subdomains with massdns + static wordlist | active, aggressive, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-24 | -| dnsbrute_mutations | scan | No | Brute-force subdomains with massdns + target-specific mutations | active, aggressive, slow, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-25 | -| dnscommonsrv | scan | No | Check for common SRV records | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-15 | -| dotnetnuke | scan | No | Scan for critical DotNetNuke (DNN) vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE | TECHNOLOGY, VULNERABILITY | @liquidsec | 2023-11-21 | -| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | @liquidsec | 2022-04-10 | -| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | @liquidsec | 2022-07-05 | -| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, safe, web-basic | HTTP_RESPONSE, URL_UNVERIFIED | FILESYSTEM | @TheTechromancer | 2023-10-11 | -| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | @TheTechromancer | 2023-01-30 | -| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | @liquidsec | 2022-07-30 | -| git | scan | No | Check for exposed .git repositories | active, code-enum, safe, web-basic | URL | FINDING | @TheTechromancer | 2023-05-30 | -| gitlab | scan | No | Detect GitLab instances and query them for repositories | active, code-enum, safe | HTTP_RESPONSE, SOCIAL, TECHNOLOGY | CODE_REPOSITORY, FINDING, SOCIAL, TECHNOLOGY | @TheTechromancer | 2024-03-11 | -| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | SOCIAL, URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | @TheTechromancer | 2022-07-08 | -| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | @liquidsec | 2022-07-27 | -| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | @TheTechromancer | 2022-07-08 | -| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-thorough | WEB_PARAMETER | FINDING | @liquidsec | 2022-07-20 | -| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic | URL | URL_HINT | @liquidsec | 2022-04-15 | -| newsletters | scan | No | Searches for Newsletter Submission Entry Fields on Websites | active, safe | HTTP_RESPONSE | FINDING | @stryker2k2 | 2024-02-02 | -| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic | HTTP_RESPONSE, URL | DNS_NAME, FINDING | @liquidsec | 2022-07-25 | -| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2022-03-12 | -| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | @TheTechromancer | 2023-07-12 | -| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | FINDING, WEB_PARAMETER | @liquidsec | 2022-06-27 | -| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | FINDING, WEB_PARAMETER | @liquidsec | 2022-06-28 | -| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | WEB_PARAMETER | @liquidsec | 2022-04-15 | -| portscan | scan | No | Port scan with masscan. By default, scans top 100 ports. | active, portscan, safe | DNS_NAME, IP_ADDRESS, IP_RANGE | OPEN_TCP_PORT | @TheTechromancer | 2024-05-15 | -| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic | URL | URL_UNVERIFIED | @liquidsec | 2023-02-01 | -| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic | HTTP_RESPONSE | FINDING | @TheTechromancer | 2023-03-17 | -| securitytxt | scan | No | Check for security.txt content | active, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME | EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 | -| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | @liquidsec | 2022-07-06 | -| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | @TheTechromancer | 2022-03-30 | -| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2022-04-10 | -| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-09-27 | -| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | @liquidsec | 2022-05-02 | -| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | @liquidsec | 2023-02-15 | -| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic | HTTP_RESPONSE | TECHNOLOGY | @liquidsec | 2022-04-15 | -| wpscan | scan | No | Wordpress security scanner. Highly recommended to use an API key for better results. | active, aggressive | HTTP_RESPONSE, TECHNOLOGY | FINDING, TECHNOLOGY, URL_UNVERIFIED, VULNERABILITY | @domwhewell-sage | 2024-05-29 | -| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | @TheTechromancer | 2022-07-25 | -| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-10-04 | -| apkpure | scan | No | Download android applications from apkpure.com | code-enum, passive, safe | MOBILE_APP | FILESYSTEM | @domwhewell-sage | 2024-10-11 | -| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | @TheTechromancer | 2022-07-25 | -| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | @TheTechromancer | 2023-07-12 | -| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-07-04 | -| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @alt-glitch | 2022-10-26 | -| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-17 | -| bucket_file_enum | scan | No | Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS, DigitalOcean | cloud-enum, passive, safe | STORAGE_BUCKET | URL_UNVERIFIED | @TheTechromancer | 2023-11-14 | -| bufferoverrun | scan | Yes | Query BufferOverrun's TLS API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-10-23 | -| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-23 | -| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 | -| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-04 | -| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 | -| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-14 | -| code_repository | scan | No | Look for code repository links in webpages | code-enum, passive, safe | URL_UNVERIFIED | CODE_REPOSITORY | @domwhewell-sage | 2024-05-15 | -| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-06-01 | -| credshed | scan | Yes | Send queries to your own credshed server to check for known credentials of your targets | passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 | -| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-13 | -| dehashed | scan | Yes | Execute queries against dehashed.com for exposed credentials | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 | -| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-25 | -| dnscaa | scan | No | Check for CAA records | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 | -| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-03-12 | -| docker_pull | scan | No | Download images from a docker repository | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 | -| dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | code-enum, passive, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | @domwhewell-sage | 2024-03-12 | -| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | -| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | -| git_clone | scan | No | Clone code github repositories | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 | -| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enum, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | @domwhewell-sage | 2023-12-14 | -| github_org | scan | No | Query Github's API for organization and member repositories | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2023-12-14 | -| github_workflows | scan | No | Download a github repositories workflow logs and workflow artifacts | code-enum, passive, safe | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-04-29 | -| google_playstore | scan | No | Search for android applications on play.google.com | code-enum, passive, safe | CODE_REPOSITORY, ORG_STUB | MOBILE_APP | @domwhewell-sage | 2024-10-08 | -| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 | -| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @TheTechromancer | 2022-04-25 | -| internetdb | scan | No | Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities | passive, portscan, safe, subdomain-enum | DNS_NAME, IP_ADDRESS | DNS_NAME, FINDING, OPEN_TCP_PORT, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2023-12-22 | -| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | @TheTechromancer | 2023-09-12 | -| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | @TheTechromancer | 2022-06-08 | -| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | @tycoonslive | 2022-11-26 | -| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-11 | -| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-10 | -| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | -| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-08 | -| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-08-10 | -| postman | scan | No | Query Postman's API for related workspaces, collections, requests and download them | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2024-09-07 | -| postman_download | scan | No | Download workspaces, collections, requests from Postman | code-enum, passive, safe, subdomain-enum | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-09-07 | -| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | -| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 | -| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 | -| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-08-04 | -| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | -| social | scan | No | Look for social media links in webpages | passive, safe, social-enum | URL_UNVERIFIED | SOCIAL | @TheTechromancer | 2023-03-28 | -| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-26 | -| subdomainradar | scan | Yes | Query the Subdomain API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 | -| trickest | scan | Yes | Query Trickest's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @amiremami | 2024-07-27 | -| trufflehog | scan | No | TruffleHog is a tool for finding credentials | code-enum, passive, safe | CODE_REPOSITORY, FILESYSTEM | FINDING, VULNERABILITY | @domwhewell-sage | 2024-03-12 | -| unstructured | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 | -| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @TheTechromancer | 2022-06-09 | -| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-04 | -| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-25 | -| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @liquidsec | 2022-04-01 | -| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-03 | -| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | @liquidsec | 2022-09-30 | -| csv | output | No | Output to CSV | | * | | @TheTechromancer | 2022-04-07 | -| discord | output | No | Message a Discord channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | -| emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | @domwhewell-sage | 2023-12-23 | -| http | output | No | Send every event to a custom URL via a web request | | * | | @TheTechromancer | 2022-04-13 | -| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | @TheTechromancer | 2022-04-07 | -| neo4j | output | No | Output to Neo4j | | * | | @TheTechromancer | 2022-04-07 | -| python | output | No | Output via Python API | | * | | @TheTechromancer | 2022-09-13 | -| slack | output | No | Message a Slack channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | -| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | @w0Tx | 2024-02-17 | -| stdout | output | No | Output to text | | * | | | | -| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | @TheTechromancer | 2023-07-31 | -| teams | output | No | Message a Teams channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | -| txt | output | No | Output to text | | * | | | | -| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | @liquidsec | 2023-02-08 | -| websocket | output | No | Output to websockets | | * | | @TheTechromancer | 2022-04-15 | -| cloudcheck | internal | No | Tag events by cloud provider, identify cloud resources like storage buckets | | * | | | | -| dnsresolve | internal | No | | | * | | | | -| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | @TheTechromancer | 2022-07-25 | -| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE, RAW_TEXT | URL_UNVERIFIED, WEB_PARAMETER | @liquidsec | 2022-06-27 | -| speculate | internal | No | Derive certain event types from others by common sense | passive | AZURE_TENANT, DNS_NAME, DNS_NAME_UNRESOLVED, FILESYSTEM, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, SOCIAL, STORAGE_BUCKET, URL, URL_UNVERIFIED, USERNAME | DNS_NAME, FILESYSTEM, FINDING, IP_ADDRESS, OPEN_TCP_PORT, ORG_STUB | @liquidsec | 2022-05-03 | +| Module | Type | Needs API Key | Description | Flags | Consumed Events | Produced Events | Author | Created Date | +|----------------------|----------|-----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------|------------------|----------------| +| ajaxpro | scan | No | Check for potentially vulnerable Ajaxpro instances | active, safe, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2024-01-18 | +| baddns | scan | No | Check hosts for domain/subdomain takeovers | active, baddns, cloud-enum, safe, subdomain-hijack, web-basic | DNS_NAME, DNS_NAME_UNRESOLVED | FINDING, VULNERABILITY | @liquidsec | 2024-01-18 | +| baddns_direct | scan | No | Check for unusual subdomain / service takeover edge cases that require direct detection | active, baddns, cloud-enum, safe, subdomain-enum | STORAGE_BUCKET, URL | FINDING, VULNERABILITY | @liquidsec | 2024-01-29 | +| baddns_zone | scan | No | Check hosts for DNS zone transfers and NSEC walks | active, baddns, cloud-enum, safe, subdomain-enum | DNS_NAME | FINDING, VULNERABILITY | @liquidsec | 2024-01-29 | +| badsecrets | scan | No | Library for detecting known or weak secrets across many web frameworks | active, safe, web-basic | HTTP_RESPONSE | FINDING, TECHNOLOGY, VULNERABILITY | @liquidsec | 2022-11-19 | +| bucket_amazon | scan | No | Check for S3 buckets related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | +| bucket_azure | scan | No | Check for Azure storage blobs related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | +| bucket_digitalocean | scan | No | Check for DigitalOcean spaces related to target | active, cloud-enum, safe, slow, web-thorough | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-08 | +| bucket_firebase | scan | No | Check for open Firebase databases related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2023-03-20 | +| bucket_google | scan | No | Check for Google object storage related to target | active, cloud-enum, safe, web-basic | DNS_NAME, STORAGE_BUCKET | FINDING, STORAGE_BUCKET | @TheTechromancer | 2022-11-04 | +| bypass403 | scan | No | Check 403 pages for common bypasses | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-07-05 | +| dastardly | scan | No | Lightweight web application security scanner | active, aggressive, deadly, slow, web-thorough | HTTP_RESPONSE | FINDING, VULNERABILITY | @domwhewell-sage | 2023-12-11 | +| dnsbrute | scan | No | Brute-force subdomains with massdns + static wordlist | active, aggressive, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-24 | +| dnsbrute_mutations | scan | No | Brute-force subdomains with massdns + target-specific mutations | active, aggressive, slow, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-04-25 | +| dnscommonsrv | scan | No | Check for common SRV records | active, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-15 | +| dotnetnuke | scan | No | Scan for critical DotNetNuke (DNN) vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE | TECHNOLOGY, VULNERABILITY | @liquidsec | 2023-11-21 | +| ffuf | scan | No | A fast web fuzzer written in Go | active, aggressive, deadly | URL | URL_UNVERIFIED | @liquidsec | 2022-04-10 | +| ffuf_shortnames | scan | No | Use ffuf in combination IIS shortnames | active, aggressive, iis-shortnames, web-thorough | URL_HINT | URL_UNVERIFIED | @liquidsec | 2022-07-05 | +| filedownload | scan | No | Download common filetypes such as PDF, DOCX, PPTX, etc. | active, safe, web-basic | HTTP_RESPONSE, URL_UNVERIFIED | FILESYSTEM | @TheTechromancer | 2023-10-11 | +| fingerprintx | scan | No | Fingerprint exposed services like RDP, SSH, MySQL, etc. | active, safe, service-enum, slow | OPEN_TCP_PORT | PROTOCOL | @TheTechromancer | 2023-01-30 | +| generic_ssrf | scan | No | Check for generic SSRFs | active, aggressive, web-thorough | URL | VULNERABILITY | @liquidsec | 2022-07-30 | +| git | scan | No | Check for exposed .git repositories | active, code-enum, safe, web-basic | URL | FINDING | @TheTechromancer | 2023-05-30 | +| gitlab | scan | No | Detect GitLab instances and query them for repositories | active, code-enum, safe | HTTP_RESPONSE, SOCIAL, TECHNOLOGY | CODE_REPOSITORY, FINDING, SOCIAL, TECHNOLOGY | @TheTechromancer | 2024-03-11 | +| gowitness | scan | No | Take screenshots of webpages | active, safe, web-screenshots | SOCIAL, URL | TECHNOLOGY, URL, URL_UNVERIFIED, WEBSCREENSHOT | @TheTechromancer | 2022-07-08 | +| host_header | scan | No | Try common HTTP Host header spoofing techniques | active, aggressive, web-thorough | HTTP_RESPONSE | FINDING | @liquidsec | 2022-07-27 | +| httpx | scan | No | Visit webpages. Many other modules rely on httpx | active, cloud-enum, safe, social-enum, subdomain-enum, web-basic | OPEN_TCP_PORT, URL, URL_UNVERIFIED | HTTP_RESPONSE, URL | @TheTechromancer | 2022-07-08 | +| hunt | scan | No | Watch for commonly-exploitable HTTP parameters | active, safe, web-thorough | WEB_PARAMETER | FINDING | @liquidsec | 2022-07-20 | +| iis_shortnames | scan | No | Check for IIS shortname vulnerability | active, iis-shortnames, safe, web-basic | URL | URL_HINT | @liquidsec | 2022-04-15 | +| newsletters | scan | No | Searches for Newsletter Submission Entry Fields on Websites | active, safe | HTTP_RESPONSE | FINDING | @stryker2k2 | 2024-02-02 | +| ntlm | scan | No | Watch for HTTP endpoints that support NTLM authentication | active, safe, web-basic | HTTP_RESPONSE, URL | DNS_NAME, FINDING | @liquidsec | 2022-07-25 | +| nuclei | scan | No | Fast and customisable vulnerability scanner | active, aggressive, deadly | URL | FINDING, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2022-03-12 | +| oauth | scan | No | Enumerate OAUTH and OpenID Connect services | active, affiliates, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME, URL_UNVERIFIED | DNS_NAME | @TheTechromancer | 2023-07-12 | +| paramminer_cookies | scan | No | Smart brute-force to check for common HTTP cookie parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | FINDING, WEB_PARAMETER | @liquidsec | 2022-06-27 | +| paramminer_getparams | scan | No | Use smart brute-force to check for common HTTP GET parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | FINDING, WEB_PARAMETER | @liquidsec | 2022-06-28 | +| paramminer_headers | scan | No | Use smart brute-force to check for common HTTP header parameters | active, aggressive, slow, web-paramminer | HTTP_RESPONSE, WEB_PARAMETER | WEB_PARAMETER | @liquidsec | 2022-04-15 | +| portscan | scan | No | Port scan with masscan. By default, scans top 100 ports. | active, portscan, safe | DNS_NAME, IP_ADDRESS, IP_RANGE | OPEN_TCP_PORT | @TheTechromancer | 2024-05-15 | +| robots | scan | No | Look for and parse robots.txt | active, safe, web-basic | URL | URL_UNVERIFIED | @liquidsec | 2023-02-01 | +| secretsdb | scan | No | Detect common secrets with secrets-patterns-db | active, safe, web-basic | HTTP_RESPONSE | FINDING | @TheTechromancer | 2023-03-17 | +| securitytxt | scan | No | Check for security.txt content | active, cloud-enum, safe, subdomain-enum, web-basic | DNS_NAME | EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 | +| smuggler | scan | No | Check for HTTP smuggling | active, aggressive, slow, web-thorough | URL | FINDING | @liquidsec | 2022-07-06 | +| sslcert | scan | No | Visit open ports and retrieve SSL certificates | active, affiliates, email-enum, safe, subdomain-enum, web-basic | OPEN_TCP_PORT | DNS_NAME, EMAIL_ADDRESS | @TheTechromancer | 2022-03-30 | +| telerik | scan | No | Scan for critical Telerik vulnerabilities | active, aggressive, web-thorough | HTTP_RESPONSE, URL | FINDING, VULNERABILITY | @liquidsec | 2022-04-10 | +| url_manipulation | scan | No | Attempt to identify URL parsing/routing based vulnerabilities | active, aggressive, web-thorough | URL | FINDING | @liquidsec | 2022-09-27 | +| vhost | scan | No | Fuzz for virtual hosts | active, aggressive, deadly, slow | URL | DNS_NAME, VHOST | @liquidsec | 2022-05-02 | +| wafw00f | scan | No | Web Application Firewall Fingerprinting Tool | active, aggressive | URL | WAF | @liquidsec | 2023-02-15 | +| wappalyzer | scan | No | Extract technologies from web responses | active, safe, web-basic | HTTP_RESPONSE | TECHNOLOGY | @liquidsec | 2022-04-15 | +| wpscan | scan | No | Wordpress security scanner. Highly recommended to use an API key for better results. | active, aggressive | HTTP_RESPONSE, TECHNOLOGY | FINDING, TECHNOLOGY, URL_UNVERIFIED, VULNERABILITY | @domwhewell-sage | 2024-05-29 | +| affiliates | scan | No | Summarize affiliate domains at the end of a scan | affiliates, passive, report, safe | * | | @TheTechromancer | 2022-07-25 | +| anubisdb | scan | No | Query jldc.me's database for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-10-04 | +| apkpure | scan | No | Download android applications from apkpure.com | code-enum, passive, safe | MOBILE_APP | FILESYSTEM | @domwhewell-sage | 2024-10-11 | +| asn | scan | No | Query ripe and bgpview.io for ASNs | passive, report, safe, subdomain-enum | IP_ADDRESS | ASN | @TheTechromancer | 2022-07-25 | +| azure_realm | scan | No | Retrieves the "AuthURL" from login.microsoftonline.com/getuserrealm | affiliates, cloud-enum, passive, safe, subdomain-enum, web-basic | DNS_NAME | URL_UNVERIFIED | @TheTechromancer | 2023-07-12 | +| azure_tenant | scan | No | Query Azure for tenant sister domains | affiliates, cloud-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-07-04 | +| bevigil | scan | Yes | Retrieve OSINT data from mobile applications using BeVigil | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @alt-glitch | 2022-10-26 | +| binaryedge | scan | Yes | Query the BinaryEdge API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-17 | +| bucket_file_enum | scan | No | Works in conjunction with the filedownload module to download files from open storage buckets. Currently supported cloud providers: AWS, DigitalOcean | cloud-enum, passive, safe | STORAGE_BUCKET | URL_UNVERIFIED | @TheTechromancer | 2023-11-14 | +| bufferoverrun | scan | Yes | Query BufferOverrun's TLS API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2024-10-23 | +| builtwith | scan | Yes | Query Builtwith.com for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-23 | +| c99 | scan | Yes | Query the C99 API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 | +| censys | scan | Yes | Query the Censys API | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-04 | +| certspotter | scan | No | Query Certspotter's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 | +| chaos | scan | Yes | Query ProjectDiscovery's Chaos API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-14 | +| code_repository | scan | No | Look for code repository links in webpages | code-enum, passive, safe | URL_UNVERIFIED | CODE_REPOSITORY | @domwhewell-sage | 2024-05-15 | +| columbus | scan | No | Query the Columbus Project API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-06-01 | +| credshed | scan | Yes | Send queries to your own credshed server to check for known credentials of your targets | passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 | +| crt | scan | No | Query crt.sh (certificate transparency) for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-05-13 | +| dehashed | scan | Yes | Execute queries against dehashed.com for exposed credentials | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS, HASHED_PASSWORD, PASSWORD, USERNAME | @SpamFaux | 2023-10-12 | +| digitorus | scan | No | Query certificatedetails.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-25 | +| dnscaa | scan | No | Check for CAA records | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @colin-stubbs | 2024-05-26 | +| dnsdumpster | scan | No | Query dnsdumpster for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-03-12 | +| docker_pull | scan | No | Download images from a docker repository | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 | +| dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | code-enum, passive, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | @domwhewell-sage | 2024-03-12 | +| emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | +| fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | +| git_clone | scan | No | Clone code github repositories | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 | +| github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enum, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | @domwhewell-sage | 2023-12-14 | +| github_org | scan | No | Query Github's API for organization and member repositories | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2023-12-14 | +| github_workflows | scan | No | Download a github repositories workflow logs and workflow artifacts | code-enum, passive, safe | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-04-29 | +| google_playstore | scan | No | Search for android applications on play.google.com | code-enum, passive, safe | CODE_REPOSITORY, ORG_STUB | MOBILE_APP | @domwhewell-sage | 2024-10-08 | +| hackertarget | scan | No | Query the hackertarget.com API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-28 | +| hunterio | scan | Yes | Query hunter.io for emails | email-enum, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, EMAIL_ADDRESS, URL_UNVERIFIED | @TheTechromancer | 2022-04-25 | +| internetdb | scan | No | Query Shodan's InternetDB for open ports, hostnames, technologies, and vulnerabilities | passive, portscan, safe, subdomain-enum | DNS_NAME, IP_ADDRESS | DNS_NAME, FINDING, OPEN_TCP_PORT, TECHNOLOGY, VULNERABILITY | @TheTechromancer | 2023-12-22 | +| ip2location | scan | Yes | Query IP2location.io's API for geolocation information. | passive, safe | IP_ADDRESS | GEOLOCATION | @TheTechromancer | 2023-09-12 | +| ipneighbor | scan | No | Look beside IPs in their surrounding subnet | aggressive, passive, subdomain-enum | IP_ADDRESS | IP_ADDRESS | @TheTechromancer | 2022-06-08 | +| ipstack | scan | Yes | Query IPStack's GeoIP API | passive, safe | IP_ADDRESS | GEOLOCATION | @tycoonslive | 2022-11-26 | +| leakix | scan | No | Query leakix.net for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-11 | +| myssl | scan | No | Query myssl.com's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-10 | +| otx | scan | No | Query otx.alienvault.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | +| passivetotal | scan | Yes | Query the PassiveTotal API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-08 | +| pgp | scan | No | Query common PGP servers for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-08-10 | +| postman | scan | No | Query Postman's API for related workspaces, collections, requests and download them | code-enum, passive, safe, subdomain-enum | ORG_STUB, SOCIAL | CODE_REPOSITORY | @domwhewell-sage | 2024-09-07 | +| postman_download | scan | No | Download workspaces, collections, requests from Postman | code-enum, passive, safe, subdomain-enum | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-09-07 | +| rapiddns | scan | No | Query rapiddns.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | +| securitytrails | scan | Yes | Query the SecurityTrails API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 | +| shodan_dns | scan | Yes | Query Shodan for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-03 | +| sitedossier | scan | No | Query sitedossier.com for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-08-04 | +| skymem | scan | No | Query skymem.info for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | +| social | scan | No | Look for social media links in webpages | passive, safe, social-enum | URL_UNVERIFIED | SOCIAL | @TheTechromancer | 2023-03-28 | +| subdomaincenter | scan | No | Query subdomain.center's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2023-07-26 | +| subdomainradar | scan | Yes | Query the Subdomain API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 | +| trickest | scan | Yes | Query Trickest's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @amiremami | 2024-07-27 | +| trufflehog | scan | No | TruffleHog is a tool for finding credentials | code-enum, passive, safe | CODE_REPOSITORY, FILESYSTEM | FINDING, VULNERABILITY | @domwhewell-sage | 2024-03-12 | +| unstructured | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 | +| urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @TheTechromancer | 2022-06-09 | +| viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-04 | +| virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-25 | +| wayback | scan | No | Query archive.org's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @liquidsec | 2022-04-01 | +| zoomeye | scan | Yes | Query ZoomEye's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-03 | +| asset_inventory | output | No | Merge hosts, open ports, technologies, findings, etc. into a single asset inventory CSV | | DNS_NAME, FINDING, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, TECHNOLOGY, URL, VULNERABILITY, WAF | IP_ADDRESS, OPEN_TCP_PORT | @liquidsec | 2022-09-30 | +| csv | output | No | Output to CSV | | * | | @TheTechromancer | 2022-04-07 | +| discord | output | No | Message a Discord channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | +| emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | @domwhewell-sage | 2023-12-23 | +| http | output | No | Send every event to a custom URL via a web request | | * | | @TheTechromancer | 2022-04-13 | +| json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | @TheTechromancer | 2022-04-07 | +| neo4j | output | No | Output to Neo4j | | * | | @TheTechromancer | 2022-04-07 | +| python | output | No | Output via Python API | | * | | @TheTechromancer | 2022-09-13 | +| slack | output | No | Message a Slack channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | +| splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | @w0Tx | 2024-02-17 | +| stdout | output | No | Output to text | | * | | | | +| subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | @TheTechromancer | 2023-07-31 | +| teams | output | No | Message a Teams channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | +| txt | output | No | Output to text | | * | | | | +| web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | @liquidsec | 2023-02-08 | +| websocket | output | No | Output to websockets | | * | | @TheTechromancer | 2022-04-15 | +| cloudcheck | internal | No | Tag events by cloud provider, identify cloud resources like storage buckets | | * | | | | +| dnsresolve | internal | No | | | * | | | | +| aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | @TheTechromancer | 2022-07-25 | +| excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE, RAW_TEXT | URL_UNVERIFIED, WEB_PARAMETER | @liquidsec | 2022-06-27 | +| speculate | internal | No | Derive certain event types from others by common sense | passive | AZURE_TENANT, DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, SOCIAL, STORAGE_BUCKET, URL, URL_UNVERIFIED, USERNAME | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, ORG_STUB | @liquidsec | 2022-05-03 | For a list of module config options, see [Module Options](../scanning/configuration.md#module-config-options). diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index a89c927aa3..00df4d4e28 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -69,14 +69,14 @@ Presets: Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] - Modules to enable. Choices: robots,leakix,docker_pull,oauth,zoomeye,paramminer_cookies,virustotal,git_clone,shodan_dns,emailformat,azure_realm,fullhunt,ffuf,pgp,dastardly,smuggler,viewdns,dotnetnuke,ipstack,host_header,passivetotal,postman,ntlm,censys,hunt,dnscaa,nuclei,vhost,bufferoverrun,github_workflows,wappalyzer,dnsbrute_mutations,urlscan,baddns_zone,trickest,apkpure,generic_ssrf,httpx,internetdb,bucket_azure,crt,bucket_file_enum,bucket_firebase,digitorus,unstructured,code_repository,portscan,paramminer_getparams,baddns_direct,github_codesearch,builtwith,chaos,url_manipulation,otx,bypass403,dnsdumpster,ajaxpro,subdomainradar,iis_shortnames,affiliates,paramminer_headers,anubisdb,bucket_google,securitytxt,secretsdb,bucket_digitalocean,hackertarget,postman_download,securitytrails,hunterio,subdomaincenter,social,ip2location,credshed,dnscommonsrv,github_org,baddns,azure_tenant,dnsbrute,sslcert,myssl,binaryedge,filedownload,fingerprintx,ipneighbor,telerik,dockerhub,newsletters,gowitness,asn,rapiddns,ffuf_shortnames,dehashed,google_playstore,bucket_amazon,sitedossier,wpscan,trufflehog,badsecrets,gitlab,skymem,bevigil,git,columbus,certspotter,wafw00f,wayback,c99 + Modules to enable. Choices: github_org,binaryedge,columbus,credshed,fingerprintx,robots,postman_download,emailformat,fullhunt,azure_tenant,git,trickest,subdomaincenter,nuclei,dnsbrute,skymem,azure_realm,dockerhub,github_codesearch,viewdns,baddns_zone,paramminer_getparams,url_manipulation,httpx,securitytxt,vhost,ipstack,postman,trufflehog,secretsdb,github_workflows,bevigil,hunterio,passivetotal,unstructured,ajaxpro,bufferoverrun,bucket_file_enum,asn,sitedossier,chaos,paramminer_cookies,censys,securitytrails,builtwith,pgp,wpscan,ntlm,filedownload,code_repository,myssl,baddns_direct,sslcert,bucket_google,certspotter,host_header,hunt,gowitness,bypass403,dastardly,docker_pull,paramminer_headers,oauth,bucket_digitalocean,anubisdb,affiliates,portscan,telerik,smuggler,shodan_dns,baddns,ip2location,zoomeye,google_playstore,urlscan,c99,hackertarget,crt,dehashed,subdomainradar,wappalyzer,apkpure,bucket_amazon,social,otx,dnscaa,badsecrets,dnscommonsrv,leakix,iis_shortnames,wafw00f,generic_ssrf,dnsdumpster,bucket_firebase,ffuf,ffuf_shortnames,internetdb,git_clone,rapiddns,newsletters,gitlab,virustotal,bucket_azure,wayback,digitorus,dotnetnuke,dnsbrute_mutations,ipneighbor -l, --list-modules List available modules. -lmo, --list-module-options Show all module config options -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. -f FLAG [FLAG ...], --flags FLAG [FLAG ...] - Enable modules by flag. Choices: aggressive,subdomain-enum,iis-shortnames,code-enum,report,subdomain-hijack,affiliates,portscan,web-thorough,active,passive,deadly,baddns,email-enum,cloud-enum,slow,web-paramminer,service-enum,web-screenshots,social-enum,web-basic,safe + Enable modules by flag. Choices: deadly,aggressive,code-enum,affiliates,portscan,slow,web-paramminer,baddns,web-screenshots,subdomain-hijack,subdomain-enum,report,service-enum,email-enum,social-enum,cloud-enum,safe,iis-shortnames,active,web-thorough,web-basic,passive -lf, --list-flags List available flags. -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...] Only enable modules with these flags (e.g. -rf passive) @@ -101,7 +101,7 @@ Output: -o DIR, --output-dir DIR Directory to output scan results -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] - Output module(s). Choices: websocket,teams,slack,asset_inventory,neo4j,web_report,python,stdout,subdomains,json,http,emails,discord,csv,splunk,txt + Output module(s). Choices: asset_inventory,discord,csv,teams,python,http,splunk,websocket,txt,emails,stdout,neo4j,json,slack,subdomains,web_report --json, -j Output scan data in JSON format --brief, -br Output only the data itself --event-types EVENT_TYPES [EVENT_TYPES ...] diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index dc2326764c..f743e41a67 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -482,7 +482,6 @@ Many modules accept their own configuration options. These options have the abil | modules.excavate.custom_yara_rules | str | Include custom Yara rules | | | modules.excavate.retain_querystring | bool | Keep the querystring intact on emitted WEB_PARAMETERS | False | | modules.excavate.yara_max_match_data | int | Sets the maximum amount of text that can extracted from a YARA regex | 2000 | -| modules.speculate.ignore_folders | list | Subfolders to ignore when crawling downloaded folders | ['.git'] | | modules.speculate.max_hosts | int | Max number of IP_RANGE hosts to convert into IP_ADDRESS events | 65536 | | modules.speculate.ports | str | The set of ports to speculate on | 80,443 | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 5f909740c0..691b7303a7 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -113,7 +113,7 @@ Below is a full list of event types along with which modules produce/consume the | DNS_NAME | 58 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, myssl, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | | DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | | EMAIL_ADDRESS | 1 | 9 | emails | credshed, dehashed, dnscaa, emailformat, hunterio, pgp, securitytxt, skymem, sslcert | -| FILESYSTEM | 3 | 7 | speculate, trufflehog, unstructured | apkpure, docker_pull, filedownload, git_clone, github_workflows, postman_download, speculate | +| FILESYSTEM | 2 | 6 | trufflehog, unstructured | apkpure, docker_pull, filedownload, git_clone, github_workflows, postman_download | | FINDING | 2 | 29 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | | GEOLOCATION | 0 | 2 | | ip2location, ipstack | | HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | diff --git a/poetry.lock b/poetry.lock index bcf4cf2513..7bfa9312ea 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2918,13 +2918,13 @@ files = [ [[package]] name = "werkzeug" -version = "3.0.4" +version = "3.1.1" description = "The comprehensive WSGI web application library." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, - {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, + {file = "werkzeug-3.1.1-py3-none-any.whl", hash = "sha256:a71124d1ef06008baafa3d266c02f56e1836a5984afd6dd6c9230669d60d9fb5"}, + {file = "werkzeug-3.1.1.tar.gz", hash = "sha256:8cd39dfbdfc1e051965f156163e2974e52c210f130810e9ad36858f0fd3edad4"}, ] [package.dependencies]