From 2e0c1f2e935fdb2ed16cbd3058ea21f6fb3b1f64 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 7 Nov 2024 16:10:14 -0500 Subject: [PATCH 01/10] add sqlite output module --- bbot/db/sql/models.py | 147 ++++++++++++++++++ bbot/modules/output/sqlite.py | 29 ++++ bbot/modules/templates/sql.py | 89 +++++++++++ bbot/scanner/scanner.py | 2 +- .../module_tests/test_module_sqlite.py | 18 +++ mkdocs.yml | 1 + 6 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 bbot/db/sql/models.py create mode 100644 bbot/modules/output/sqlite.py create mode 100644 bbot/modules/templates/sql.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_sqlite.py diff --git a/bbot/db/sql/models.py b/bbot/db/sql/models.py new file mode 100644 index 000000000..7677a181e --- /dev/null +++ b/bbot/db/sql/models.py @@ -0,0 +1,147 @@ +# This file contains SQLModel (Pydantic + SQLAlchemy) models for BBOT events, scans, and targets. +# Used by the SQL output modules, but portable for outside use. + +import json +import logging +from datetime import datetime +from pydantic import ConfigDict +from typing import List, Optional +from typing_extensions import Annotated +from pydantic.functional_validators import AfterValidator +from sqlmodel import inspect, Column, Field, SQLModel, JSON, String, DateTime as SQLADateTime + + +log = logging.getLogger("bbot_server.models") + + +def naive_datetime_validator(d: datetime): + """ + Converts all dates into UTC, then drops timezone information. + + This is needed to prevent inconsistencies in sqlite, because it is timezone-naive. + """ + # drop timezone info + return d.replace(tzinfo=None) + + +NaiveUTC = Annotated[datetime, AfterValidator(naive_datetime_validator)] + + +class CustomJSONEncoder(json.JSONEncoder): + def default(self, obj): + # handle datetime + if isinstance(obj, datetime): + return obj.isoformat() + return super().default(obj) + + +class BBOTBaseModel(SQLModel): + model_config = ConfigDict(extra="ignore") + + def __init__(self, *args, **kwargs): + self._validated = None + super().__init__(*args, **kwargs) + + @property + def validated(self): + try: + if self._validated is None: + self._validated = self.__class__.model_validate(self) + return self._validated + except AttributeError: + return self + + def to_json(self, **kwargs): + return json.dumps(self.validated.model_dump(), sort_keys=True, cls=CustomJSONEncoder, **kwargs) + + @classmethod + def _pk_column_names(cls): + return [column.name for column in inspect(cls).primary_key] + + def __hash__(self): + return hash(self.to_json()) + + def __eq__(self, other): + return hash(self) == hash(other) + + +### EVENT ### + + +class Event(BBOTBaseModel, table=True): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + data = self._get_data(self.data, self.type) + self.data = {self.type: data} + if self.host: + self.reverse_host = self.host[::-1] + + def get_data(self): + return self._get_data(self.data, self.type) + + @staticmethod + def _get_data(data, type): + # handle SIEM-friendly format + if isinstance(data, dict) and list(data) == [type]: + return data[type] + return data + + uuid: str = Field( + primary_key=True, + index=True, + nullable=False, + ) + id: str = Field(index=True) + type: str = Field(index=True) + scope_description: str + data: dict = Field(sa_type=JSON) + host: Optional[str] + port: Optional[int] + netloc: Optional[str] + # store the host in reversed form for efficient lookups by domain + reverse_host: Optional[str] = Field(default="", exclude=True, index=True) + resolved_hosts: List = Field(default=[], sa_type=JSON) + dns_children: dict = Field(default={}, sa_type=JSON) + web_spider_distance: int = 10 + scope_distance: int = Field(default=10, index=True) + scan: str = Field(index=True) + timestamp: NaiveUTC = Field(index=True) + parent: str = Field(index=True) + tags: List = Field(default=[], sa_type=JSON) + module: str = Field(index=True) + module_sequence: str + discovery_context: str = "" + discovery_path: List[str] = Field(default=[], sa_type=JSON) + parent_chain: List[str] = Field(default=[], sa_type=JSON) + + +### SCAN ### + + +class Scan(BBOTBaseModel, table=True): + id: str = Field(primary_key=True) + name: str + status: str + started_at: NaiveUTC = Field(index=True) + finished_at: Optional[NaiveUTC] = Field(default=None, sa_column=Column(SQLADateTime, nullable=True, index=True)) + duration_seconds: Optional[float] = Field(default=None) + duration: Optional[str] = Field(default=None) + target: dict = Field(sa_type=JSON) + preset: dict = Field(sa_type=JSON) + + +### TARGET ### + + +class Target(BBOTBaseModel, table=True): + name: str = "Default Target" + strict_scope: bool = False + seeds: List = Field(default=[], sa_type=JSON) + whitelist: List = Field(default=None, sa_type=JSON) + blacklist: List = Field(default=[], sa_type=JSON) + hash: str = Field(sa_column=Column("hash", String, unique=True, primary_key=True, index=True)) + scope_hash: str = Field(sa_column=Column("scope_hash", String, index=True)) + seed_hash: str = Field(sa_column=Column("seed_hashhash", String, index=True)) + whitelist_hash: str = Field(sa_column=Column("whitelist_hash", String, index=True)) + blacklist_hash: str = Field(sa_column=Column("blacklist_hash", String, index=True)) diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py new file mode 100644 index 000000000..857687e22 --- /dev/null +++ b/bbot/modules/output/sqlite.py @@ -0,0 +1,29 @@ +from pathlib import Path + +from bbot.modules.templates.sql import SQLTemplate + + +class SQLite(SQLTemplate): + watched_events = ["*"] + meta = {"description": "sqlite"} + deps_pip = ["sqlmodel", "sqlalchemy-utils", "aiosqlite"] + options = { + "database": "", + } + options_desc = { + "database": "The path to the sqlite database file", + } + + async def setup(self): + db_file = self.config.get("database", "") + if not db_file: + db_file = self.scan.home / "output.sqlite" + db_file = Path(db_file) + if not db_file.is_absolute(): + db_file = self.scan.home / db_file + self.db_file = db_file + self.db_file.parent.mkdir(parents=True, exist_ok=True) + return await super().setup() + + def connection_string(self, mask_password=False): + return f"sqlite+aiosqlite:///{self.db_file}" diff --git a/bbot/modules/templates/sql.py b/bbot/modules/templates/sql.py new file mode 100644 index 000000000..b075753d3 --- /dev/null +++ b/bbot/modules/templates/sql.py @@ -0,0 +1,89 @@ +from sqlmodel import SQLModel +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession +from sqlalchemy_utils.functions import database_exists, create_database + +from bbot.db.sql.models import Event, Scan, Target +from bbot.modules.output.base import BaseOutputModule + + +class SQLTemplate(BaseOutputModule): + meta = {"description": "SQL output module template"} + options = { + "protocol": "", + "database": "bbot", + "username": "", + "password": "", + "host": "127.0.0.1", + "port": 0, + } + options_desc = { + "protocol": "The protocol to use to connect to the database", + "database": "The database to use", + "username": "The username to use to connect to the database", + "password": "The password to use to connect to the database", + "host": "The host to use to connect to the database", + "port": "The port to use to connect to the database", + } + + async def setup(self): + self.database = self.config.get("database", "bbot") + self.username = self.config.get("username", "") + self.password = self.config.get("password", "") + self.host = self.config.get("host", "127.0.0.1") + self.port = self.config.get("port", 0) + + self.log.info(f"Connecting to {self.connection_string(mask_password=True)}") + + self.engine = create_async_engine(self.connection_string()) + # Create a session factory bound to the engine + self.async_session = sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession) + await self.init_database() + return True + + async def handle_event(self, event): + event_obj = Event(**event.json()).validated + + async with self.async_session() as session: + async with session.begin(): + # insert event + session.add(event_obj) + + # if it's a SCAN event, create/update the scan and target + if event_obj.type == "SCAN": + event_data = event_obj.get_data() + if not isinstance(event_data, dict): + raise ValueError(f"Invalid data for SCAN event: {event_data}") + scan = Scan(**event_data).validated + await session.merge(scan) # Insert or update scan + + target_data = event_data.get("target", {}) + if not isinstance(target_data, dict): + raise ValueError(f"Invalid target for SCAN event: {target_data}") + target = Target(**target_data).validated + await session.merge(target) # Insert or update target + + await session.commit() + + async def init_database(self): + async with self.engine.begin() as conn: + # Check if the database exists using the connection's engine URL + if not await conn.run_sync(lambda sync_conn: database_exists(sync_conn.engine.url)): + await conn.run_sync(lambda sync_conn: create_database(sync_conn.engine.url)) + # Create all tables + await conn.run_sync(SQLModel.metadata.create_all) + + def connection_string(self, mask_password=False): + connection_string = f"{self.protocol}://" + if self.username: + password = self.password + if mask_password: + password = "****" + connection_string += f"{self.username}:{password}" + if self.host: + connection_string += f"@{self.host}" + if self.port: + connection_string += f":{self.port}" + if self.database: + connection_string += f"/{self.database}" + return connection_string diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 34ef29c38..957215355 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -161,7 +161,7 @@ def __init__( tries += 1 else: scan_name = str(self.preset.scan_name) - self.name = scan_name + self.name = scan_name.replace("/", "_") # make sure the preset has a description if not self.preset.description: diff --git a/bbot/test/test_step_2/module_tests/test_module_sqlite.py b/bbot/test/test_step_2/module_tests/test_module_sqlite.py new file mode 100644 index 000000000..809d68c47 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_sqlite.py @@ -0,0 +1,18 @@ +import sqlite3 +from .base import ModuleTestBase + + +class TestSQLite(ModuleTestBase): + targets = ["evilcorp.com"] + + def check(self, module_test, events): + sqlite_output_file = module_test.scan.home / "output.sqlite" + assert sqlite_output_file.exists(), "SQLite output file not found" + with sqlite3.connect(sqlite_output_file) as db: + cursor = db.cursor() + cursor.execute("SELECT * FROM event") + assert len(cursor.fetchall()) > 0, "No events found in SQLite database" + cursor.execute("SELECT * FROM scan") + assert len(cursor.fetchall()) > 0, "No scans found in SQLite database" + cursor.execute("SELECT * FROM target") + assert len(cursor.fetchall()) > 0, "No targets found in SQLite database" diff --git a/mkdocs.yml b/mkdocs.yml index a5c29b757..1802fc678 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ nav: - Modules: - List of Modules: modules/list_of_modules.md - Nuclei: modules/nuclei.md + - Custom YARA Rules: modules/custom_yara_rules.md - Misc: - Contribution: contribution.md - Release History: release_history.md From e344054721e4da97083e249b58887b2d3f43070d Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 7 Nov 2024 16:22:50 -0500 Subject: [PATCH 02/10] update docs --- bbot/modules/output/sqlite.py | 4 ++-- docs/scanning/output.md | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py index 857687e22..68ac60daf 100644 --- a/bbot/modules/output/sqlite.py +++ b/bbot/modules/output/sqlite.py @@ -5,14 +5,14 @@ class SQLite(SQLTemplate): watched_events = ["*"] - meta = {"description": "sqlite"} - deps_pip = ["sqlmodel", "sqlalchemy-utils", "aiosqlite"] + meta = {"description": "Output scan data to a SQLite database"} options = { "database": "", } options_desc = { "database": "The path to the sqlite database file", } + deps_pip = ["sqlmodel", "sqlalchemy-utils", "aiosqlite"] async def setup(self): db_file = self.config.get("database", "") diff --git a/docs/scanning/output.md b/docs/scanning/output.md index 55eaa5469..7efdf4862 100644 --- a/docs/scanning/output.md +++ b/docs/scanning/output.md @@ -178,6 +178,15 @@ The `asset_inventory` module produces a CSV like this: | www.evilcorp.com | cdn-github | 2.3.4.5 | Active | 22,80,443 | | admin.evilcorp.com | cloud-azure | 5.6.7.8 | N/A | | +### SQLite + +The `sqlite` output module produces a SQLite database containing all events, scans, and targets. By default, it will be saved in the scan directory as `output.sqlite`. + +```bash +# specifying a custom database path +bbot -t evilcorp.com -om sqlite -c modules.sqlite.database=/tmp/bbot.sqlite +``` + ### Subdomains The `subdomains` output module produces simple text file containing only in-scope and resolved subdomains: From 770bb52f2d53cd314586bdcfe47aebad3c7feb45 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 7 Nov 2024 16:27:12 -0500 Subject: [PATCH 03/10] more docs --- docs/release_history.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release_history.md b/docs/release_history.md index f93f7d785..b37fbc2aa 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,3 +1,9 @@ +### 2.1.2 +- https://github.com/blacklanternsecurity/bbot/pull/1909 + +### 2.1.1 +- https://github.com/blacklanternsecurity/bbot/pull/1885 + ### 2.1.0 - https://github.com/blacklanternsecurity/bbot/pull/1724 From 4a07535811c0da17a4a89c206492cf55ab9ee38e Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 7 Nov 2024 16:29:46 -0500 Subject: [PATCH 04/10] more docs --- docs/release_history.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/release_history.md b/docs/release_history.md index b37fbc2aa..7fd343f51 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,10 +1,10 @@ -### 2.1.2 +### 2.1.2 - Nov 1, 2024 - https://github.com/blacklanternsecurity/bbot/pull/1909 -### 2.1.1 +### 2.1.1 - Oct 31, 2024 - https://github.com/blacklanternsecurity/bbot/pull/1885 -### 2.1.0 +### 2.1.0 - Oct 18, 2024 - https://github.com/blacklanternsecurity/bbot/pull/1724 ### 2.0.1 - Aug 29, 2024 From 7ef746129cd9b920279ba390fcbf10d13fdf70f6 Mon Sep 17 00:00:00 2001 From: liquidsec Date: Fri, 8 Nov 2024 16:10:13 -0500 Subject: [PATCH 05/10] version bump badsecrets/baddns --- bbot/modules/baddns.py | 2 +- bbot/modules/baddns_direct.py | 2 +- bbot/modules/baddns_zone.py | 2 +- bbot/modules/badsecrets.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 2a5892f25..443606f7e 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -22,7 +22,7 @@ class baddns(BaseModule): "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", } module_threads = 8 - deps_pip = ["baddns~=1.1.864"] + deps_pip = ["baddns~=1.4.13"] def select_modules(self): selected_submodules = [] diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index e60c2bbb6..33b6b9575 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -19,7 +19,7 @@ class baddns_direct(BaseModule): "custom_nameservers": "Force BadDNS to use a list of custom nameservers", } module_threads = 8 - deps_pip = ["baddns~=1.1.864"] + deps_pip = ["baddns~=1.4.13"] scope_distance_modifier = 1 diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index ca3ab39b0..b8f396769 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -16,7 +16,7 @@ class baddns_zone(baddns_module): "only_high_confidence": "Do not emit low-confidence or generic detections", } module_threads = 8 - deps_pip = ["baddns~=1.1.864"] + deps_pip = ["baddns~=1.4.13"] def set_modules(self): self.enabled_submodules = ["NSEC", "zonetransfer"] diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 0295bfc2c..f2d5a092f 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -17,7 +17,7 @@ class badsecrets(BaseModule): options_desc = { "custom_secrets": "Include custom secrets loaded from a local file", } - deps_pip = ["badsecrets~=0.4.490"] + deps_pip = ["badsecrets~=0.6.21"] async def setup(self): self.custom_secrets = None From ed03047a1e02cf721c9a616bf09ccbbc42647c2c Mon Sep 17 00:00:00 2001 From: TheTechromancer <20261699+TheTechromancer@users.noreply.github.com> Date: Sun, 10 Nov 2024 02:45:17 +0000 Subject: [PATCH 06/10] [create-pull-request] automated change --- README.md | 1 + docs/data/chord_graph/entities.json | 448 ++++++++++++++-------------- docs/data/chord_graph/rels.json | 372 +++++++++++------------ docs/modules/list_of_modules.md | 3 +- docs/scanning/advanced.md | 6 +- docs/scanning/configuration.md | 5 +- docs/scanning/events.md | 6 +- docs/scanning/index.md | 48 +-- 8 files changed, 446 insertions(+), 443 deletions(-) diff --git a/README.md b/README.md index ad323ce79..50e26da26 100644 --- a/README.md +++ b/README.md @@ -357,6 +357,7 @@ For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/Stabl - **Modules** - [List of Modules](https://www.blacklanternsecurity.com/bbot/Stable/modules/list_of_modules) - [Nuclei](https://www.blacklanternsecurity.com/bbot/Stable/modules/nuclei) + - [Custom YARA Rules](https://www.blacklanternsecurity.com/bbot/Stable/modules/custom_yara_rules) - **Misc** - [Contribution](https://www.blacklanternsecurity.com/bbot/Stable/contribution) - [Release History](https://www.blacklanternsecurity.com/bbot/Stable/release_history) diff --git a/docs/data/chord_graph/entities.json b/docs/data/chord_graph/entities.json index 2d8baf148..d6df50b93 100644 --- a/docs/data/chord_graph/entities.json +++ b/docs/data/chord_graph/entities.json @@ -23,11 +23,11 @@ ] }, { - "id": 124, + "id": 125, "name": "AZURE_TENANT", "parent": 88888888, "consumes": [ - 123 + 124 ], "produces": [] }, @@ -37,19 +37,19 @@ "parent": 88888888, "consumes": [ 59, - 78, - 81, - 83, - 112, - 131 + 79, + 82, + 84, + 113, + 132 ], "produces": [ 42, 60, - 79, 80, - 82, - 111 + 81, + 83, + 112 ] }, { @@ -87,29 +87,29 @@ 57, 58, 64, - 75, - 79, - 86, - 90, - 92, - 97, + 76, + 80, + 87, + 91, + 93, 98, - 102, + 99, 103, - 107, + 104, 108, 109, - 113, - 116, + 110, + 114, 117, 118, 119, 120, - 123, - 126, + 121, + 124, 127, 128, - 130, + 129, + 131, 134, 137, 138, @@ -135,25 +135,25 @@ 56, 57, 58, - 75, - 86, - 90, - 92, - 97, + 76, + 87, + 91, + 93, 98, - 100, - 102, + 99, + 101, 103, - 107, - 113, - 116, - 118, + 104, + 108, + 114, + 117, 119, - 123, - 125, + 120, + 124, 126, 127, - 130, + 128, + 131, 134, 135, 137, @@ -168,8 +168,8 @@ "parent": 88888888, "consumes": [ 21, - 123, - 128 + 124, + 129 ], "produces": [] }, @@ -185,11 +185,11 @@ 52, 56, 64, - 90, - 108, - 117, - 120, - 125 + 91, + 109, + 118, + 121, + 126 ] }, { @@ -197,16 +197,16 @@ "name": "FILESYSTEM", "parent": 88888888, "consumes": [ - 131, + 69, 132 ], "produces": [ 8, 59, - 72, - 78, - 81, - 112 + 73, + 79, + 82, + 113 ] }, { @@ -230,33 +230,33 @@ 34, 37, 51, - 77, - 82, - 87, - 89, - 92, - 99, + 78, + 83, + 88, + 90, + 93, 100, 101, - 104, + 102, 105, - 115, - 121, - 123, - 129, - 131, + 106, + 116, + 122, + 124, + 130, + 132, 133, 143 ] }, { - "id": 94, + "id": 95, "name": "GEOLOCATION", "parent": 88888888, "consumes": [], "produces": [ - 93, - 96 + 94, + 97 ] }, { @@ -280,22 +280,22 @@ 51, 63, 66, - 72, - 82, - 87, - 99, + 73, + 83, + 88, 100, - 104, + 101, 105, 106, - 115, - 123, - 129, + 107, + 116, + 124, + 130, 140, 143 ], "produces": [ - 88 + 89 ] }, { @@ -305,26 +305,26 @@ "consumes": [ 11, 14, - 92, 93, - 95, + 94, 96, - 109, - 123 + 97, + 110, + 124 ], "produces": [ 14, - 95, - 123 + 96, + 124 ] }, { - "id": 110, + "id": 111, "name": "IP_RANGE", "parent": 88888888, "consumes": [ - 109, - 123 + 110, + 124 ], "produces": [] }, @@ -336,7 +336,7 @@ 8 ], "produces": [ - 83 + 84 ] }, { @@ -345,15 +345,15 @@ "parent": 88888888, "consumes": [ 14, - 73, - 88, - 125 + 74, + 89, + 126 ], "produces": [ 14, - 92, - 109, - 123 + 93, + 110, + 124 ] }, { @@ -362,12 +362,12 @@ "parent": 88888888, "consumes": [ 60, - 80, - 83, - 111 + 81, + 84, + 112 ], "produces": [ - 123 + 124 ] }, { @@ -381,12 +381,12 @@ ] }, { - "id": 74, + "id": 75, "name": "PROTOCOL", "parent": 88888888, "consumes": [], "produces": [ - 73 + 74 ] }, { @@ -397,7 +397,7 @@ 66 ], "produces": [ - 132 + 69 ] }, { @@ -406,16 +406,16 @@ "parent": 88888888, "consumes": [ 60, - 80, - 82, - 84, - 111, - 123 + 81, + 83, + 85, + 112, + 124 ], "produces": [ 60, - 82, - 122 + 83, + 123 ] }, { @@ -430,7 +430,7 @@ 32, 33, 34, - 123 + 124 ], "produces": [ 29, @@ -446,17 +446,17 @@ "parent": 88888888, "consumes": [ 14, - 82, + 83, 142, 143 ], "produces": [ 26, 63, - 82, - 84, - 92, - 101, + 83, + 85, + 93, + 102, 140, 143 ] @@ -470,37 +470,37 @@ 14, 23, 37, - 69, - 76, + 70, 77, - 84, - 88, - 91, - 100, + 78, + 85, + 89, + 92, 101, - 114, - 121, - 123, - 129, + 102, + 115, + 122, + 124, + 130, 133, 135, 139, 142 ], "produces": [ - 84, - 88 + 85, + 89 ] }, { - "id": 71, + "id": 72, "name": "URL_HINT", "parent": 88888888, "consumes": [ - 70 + 71 ], "produces": [ - 91 + 92 ] }, { @@ -509,11 +509,11 @@ "parent": 88888888, "consumes": [ 42, - 72, - 88, - 102, - 122, - 123 + 73, + 89, + 103, + 123, + 124 ], "produces": [ 18, @@ -522,13 +522,13 @@ 56, 60, 66, - 69, 70, - 79, - 84, - 90, - 114, - 117, + 71, + 80, + 85, + 91, + 115, + 118, 134, 141, 143 @@ -539,7 +539,7 @@ "name": "USERNAME", "parent": 88888888, "consumes": [ - 123 + 124 ], "produces": [ 45, @@ -573,11 +573,11 @@ 26, 51, 63, - 76, - 92, - 101, - 129, - 131, + 77, + 93, + 102, + 130, + 132, 143 ] }, @@ -593,12 +593,12 @@ ] }, { - "id": 85, + "id": 86, "name": "WEBSCREENSHOT", "parent": 88888888, "consumes": [], "produces": [ - 84 + 85 ] }, { @@ -606,16 +606,16 @@ "name": "WEB_PARAMETER", "parent": 88888888, "consumes": [ - 89, - 104, + 90, 105, - 106 + 106, + 107 ], "produces": [ 66, - 104, 105, - 106 + 106, + 107 ] }, { @@ -1146,6 +1146,17 @@ }, { "id": 69, + "name": "extractous", + "parent": 99999999, + "consumes": [ + 10 + ], + "produces": [ + 67 + ] + }, + { + "id": 70, "name": "ffuf", "parent": 99999999, "consumes": [ @@ -1156,18 +1167,18 @@ ] }, { - "id": 70, + "id": 71, "name": "ffuf_shortnames", "parent": 99999999, "consumes": [ - 71 + 72 ], "produces": [ 19 ] }, { - "id": 72, + "id": 73, "name": "filedownload", "parent": 99999999, "consumes": [ @@ -1179,18 +1190,18 @@ ] }, { - "id": 73, + "id": 74, "name": "fingerprintx", "parent": 99999999, "consumes": [ 15 ], "produces": [ - 74 + 75 ] }, { - "id": 75, + "id": 76, "name": "fullhunt", "parent": 99999999, "consumes": [ @@ -1201,7 +1212,7 @@ ] }, { - "id": 76, + "id": 77, "name": "generic_ssrf", "parent": 99999999, "consumes": [ @@ -1212,7 +1223,7 @@ ] }, { - "id": 77, + "id": 78, "name": "git", "parent": 99999999, "consumes": [ @@ -1223,7 +1234,7 @@ ] }, { - "id": 78, + "id": 79, "name": "git_clone", "parent": 99999999, "consumes": [ @@ -1234,7 +1245,7 @@ ] }, { - "id": 79, + "id": 80, "name": "github_codesearch", "parent": 99999999, "consumes": [ @@ -1246,7 +1257,7 @@ ] }, { - "id": 80, + "id": 81, "name": "github_org", "parent": 99999999, "consumes": [ @@ -1258,7 +1269,7 @@ ] }, { - "id": 81, + "id": 82, "name": "github_workflows", "parent": 99999999, "consumes": [ @@ -1269,7 +1280,7 @@ ] }, { - "id": 82, + "id": 83, "name": "gitlab", "parent": 99999999, "consumes": [ @@ -1285,7 +1296,7 @@ ] }, { - "id": 83, + "id": 84, "name": "google_playstore", "parent": 99999999, "consumes": [ @@ -1297,7 +1308,7 @@ ] }, { - "id": 84, + "id": 85, "name": "gowitness", "parent": 99999999, "consumes": [ @@ -1308,11 +1319,11 @@ 16, 3, 19, - 85 + 86 ] }, { - "id": 86, + "id": 87, "name": "hackertarget", "parent": 99999999, "consumes": [ @@ -1323,7 +1334,7 @@ ] }, { - "id": 87, + "id": 88, "name": "host_header", "parent": 99999999, "consumes": [ @@ -1334,7 +1345,7 @@ ] }, { - "id": 88, + "id": 89, "name": "httpx", "parent": 99999999, "consumes": [ @@ -1348,7 +1359,7 @@ ] }, { - "id": 89, + "id": 90, "name": "hunt", "parent": 99999999, "consumes": [ @@ -1359,7 +1370,7 @@ ] }, { - "id": 90, + "id": 91, "name": "hunterio", "parent": 99999999, "consumes": [ @@ -1372,18 +1383,18 @@ ] }, { - "id": 91, + "id": 92, "name": "iis_shortnames", "parent": 99999999, "consumes": [ 3 ], "produces": [ - 71 + 72 ] }, { - "id": 92, + "id": 93, "name": "internetdb", "parent": 99999999, "consumes": [ @@ -1399,18 +1410,18 @@ ] }, { - "id": 93, + "id": 94, "name": "ip2location", "parent": 99999999, "consumes": [ 12 ], "produces": [ - 94 + 95 ] }, { - "id": 95, + "id": 96, "name": "ipneighbor", "parent": 99999999, "consumes": [ @@ -1421,18 +1432,18 @@ ] }, { - "id": 96, + "id": 97, "name": "ipstack", "parent": 99999999, "consumes": [ 12 ], "produces": [ - 94 + 95 ] }, { - "id": 97, + "id": 98, "name": "leakix", "parent": 99999999, "consumes": [ @@ -1443,7 +1454,7 @@ ] }, { - "id": 98, + "id": 99, "name": "myssl", "parent": 99999999, "consumes": [ @@ -1454,7 +1465,7 @@ ] }, { - "id": 99, + "id": 100, "name": "newsletters", "parent": 99999999, "consumes": [ @@ -1465,7 +1476,7 @@ ] }, { - "id": 100, + "id": 101, "name": "ntlm", "parent": 99999999, "consumes": [ @@ -1478,7 +1489,7 @@ ] }, { - "id": 101, + "id": 102, "name": "nuclei", "parent": 99999999, "consumes": [ @@ -1491,7 +1502,7 @@ ] }, { - "id": 102, + "id": 103, "name": "oauth", "parent": 99999999, "consumes": [ @@ -1503,7 +1514,7 @@ ] }, { - "id": 103, + "id": 104, "name": "otx", "parent": 99999999, "consumes": [ @@ -1514,7 +1525,7 @@ ] }, { - "id": 104, + "id": 105, "name": "paramminer_cookies", "parent": 99999999, "consumes": [ @@ -1527,7 +1538,7 @@ ] }, { - "id": 105, + "id": 106, "name": "paramminer_getparams", "parent": 99999999, "consumes": [ @@ -1540,7 +1551,7 @@ ] }, { - "id": 106, + "id": 107, "name": "paramminer_headers", "parent": 99999999, "consumes": [ @@ -1552,7 +1563,7 @@ ] }, { - "id": 107, + "id": 108, "name": "passivetotal", "parent": 99999999, "consumes": [ @@ -1563,7 +1574,7 @@ ] }, { - "id": 108, + "id": 109, "name": "pgp", "parent": 99999999, "consumes": [ @@ -1574,20 +1585,20 @@ ] }, { - "id": 109, + "id": 110, "name": "portscan", "parent": 99999999, "consumes": [ 7, 12, - 110 + 111 ], "produces": [ 15 ] }, { - "id": 111, + "id": 112, "name": "postman", "parent": 99999999, "consumes": [ @@ -1599,7 +1610,7 @@ ] }, { - "id": 112, + "id": 113, "name": "postman_download", "parent": 99999999, "consumes": [ @@ -1610,7 +1621,7 @@ ] }, { - "id": 113, + "id": 114, "name": "rapiddns", "parent": 99999999, "consumes": [ @@ -1621,7 +1632,7 @@ ] }, { - "id": 114, + "id": 115, "name": "robots", "parent": 99999999, "consumes": [ @@ -1632,7 +1643,7 @@ ] }, { - "id": 115, + "id": 116, "name": "secretsdb", "parent": 99999999, "consumes": [ @@ -1643,7 +1654,7 @@ ] }, { - "id": 116, + "id": 117, "name": "securitytrails", "parent": 99999999, "consumes": [ @@ -1654,7 +1665,7 @@ ] }, { - "id": 117, + "id": 118, "name": "securitytxt", "parent": 99999999, "consumes": [ @@ -1666,7 +1677,7 @@ ] }, { - "id": 118, + "id": 119, "name": "shodan_dns", "parent": 99999999, "consumes": [ @@ -1677,7 +1688,7 @@ ] }, { - "id": 119, + "id": 120, "name": "sitedossier", "parent": 99999999, "consumes": [ @@ -1688,7 +1699,7 @@ ] }, { - "id": 120, + "id": 121, "name": "skymem", "parent": 99999999, "consumes": [ @@ -1699,7 +1710,7 @@ ] }, { - "id": 121, + "id": 122, "name": "smuggler", "parent": 99999999, "consumes": [ @@ -1710,7 +1721,7 @@ ] }, { - "id": 122, + "id": 123, "name": "social", "parent": 99999999, "consumes": [ @@ -1721,16 +1732,16 @@ ] }, { - "id": 123, + "id": 124, "name": "speculate", "parent": 99999999, "consumes": [ - 124, + 125, 7, 22, 2, 12, - 110, + 111, 62, 24, 3, @@ -1746,7 +1757,7 @@ ] }, { - "id": 125, + "id": 126, "name": "sslcert", "parent": 99999999, "consumes": [ @@ -1758,7 +1769,7 @@ ] }, { - "id": 126, + "id": 127, "name": "subdomaincenter", "parent": 99999999, "consumes": [ @@ -1769,7 +1780,7 @@ ] }, { - "id": 127, + "id": 128, "name": "subdomainradar", "parent": 99999999, "consumes": [ @@ -1780,7 +1791,7 @@ ] }, { - "id": 128, + "id": 129, "name": "subdomains", "parent": 99999999, "consumes": [ @@ -1790,7 +1801,7 @@ "produces": [] }, { - "id": 129, + "id": 130, "name": "telerik", "parent": 99999999, "consumes": [ @@ -1803,7 +1814,7 @@ ] }, { - "id": 130, + "id": 131, "name": "trickest", "parent": 99999999, "consumes": [ @@ -1814,7 +1825,7 @@ ] }, { - "id": 131, + "id": 132, "name": "trufflehog", "parent": 99999999, "consumes": [ @@ -1826,17 +1837,6 @@ 5 ] }, - { - "id": 132, - "name": "unstructured", - "parent": 99999999, - "consumes": [ - 10 - ], - "produces": [ - 67 - ] - }, { "id": 133, "name": "url_manipulation", diff --git a/docs/data/chord_graph/rels.json b/docs/data/chord_graph/rels.json index 78ab196e1..a399b4cd7 100644 --- a/docs/data/chord_graph/rels.json +++ b/docs/data/chord_graph/rels.json @@ -656,17 +656,17 @@ }, { "source": 69, - "target": 3, + "target": 10, "type": "consumes" }, { - "source": 19, + "source": 67, "target": 69, "type": "produces" }, { "source": 70, - "target": 71, + "target": 3, "type": "consumes" }, { @@ -675,842 +675,842 @@ "type": "produces" }, { - "source": 72, + "source": 71, + "target": 72, + "type": "consumes" + }, + { + "source": 19, + "target": 71, + "type": "produces" + }, + { + "source": 73, "target": 2, "type": "consumes" }, { - "source": 72, + "source": 73, "target": 19, "type": "consumes" }, { "source": 10, - "target": 72, + "target": 73, "type": "produces" }, { - "source": 73, + "source": 74, "target": 15, "type": "consumes" }, { - "source": 74, - "target": 73, + "source": 75, + "target": 74, "type": "produces" }, { - "source": 75, + "source": 76, "target": 7, "type": "consumes" }, { "source": 7, - "target": 75, + "target": 76, "type": "produces" }, { - "source": 76, + "source": 77, "target": 3, "type": "consumes" }, { "source": 5, - "target": 76, + "target": 77, "type": "produces" }, { - "source": 77, + "source": 78, "target": 3, "type": "consumes" }, { "source": 4, - "target": 77, + "target": 78, "type": "produces" }, { - "source": 78, + "source": 79, "target": 43, "type": "consumes" }, { "source": 10, - "target": 78, + "target": 79, "type": "produces" }, { - "source": 79, + "source": 80, "target": 7, "type": "consumes" }, { "source": 43, - "target": 79, + "target": 80, "type": "produces" }, { "source": 19, - "target": 79, + "target": 80, "type": "produces" }, { - "source": 80, + "source": 81, "target": 61, "type": "consumes" }, { - "source": 80, + "source": 81, "target": 62, "type": "consumes" }, { "source": 43, - "target": 80, + "target": 81, "type": "produces" }, { - "source": 81, + "source": 82, "target": 43, "type": "consumes" }, { "source": 10, - "target": 81, + "target": 82, "type": "produces" }, { - "source": 82, + "source": 83, "target": 2, "type": "consumes" }, { - "source": 82, + "source": 83, "target": 62, "type": "consumes" }, { - "source": 82, + "source": 83, "target": 16, "type": "consumes" }, { "source": 43, - "target": 82, + "target": 83, "type": "produces" }, { "source": 4, - "target": 82, + "target": 83, "type": "produces" }, { "source": 62, - "target": 82, + "target": 83, "type": "produces" }, { "source": 16, - "target": 82, + "target": 83, "type": "produces" }, { - "source": 83, + "source": 84, "target": 43, "type": "consumes" }, { - "source": 83, + "source": 84, "target": 61, "type": "consumes" }, { "source": 9, - "target": 83, + "target": 84, "type": "produces" }, { - "source": 84, + "source": 85, "target": 62, "type": "consumes" }, { - "source": 84, + "source": 85, "target": 3, "type": "consumes" }, { "source": 16, - "target": 84, + "target": 85, "type": "produces" }, { "source": 3, - "target": 84, + "target": 85, "type": "produces" }, { "source": 19, - "target": 84, + "target": 85, "type": "produces" }, { - "source": 85, - "target": 84, + "source": 86, + "target": 85, "type": "produces" }, { - "source": 86, + "source": 87, "target": 7, "type": "consumes" }, { "source": 7, - "target": 86, + "target": 87, "type": "produces" }, { - "source": 87, + "source": 88, "target": 2, "type": "consumes" }, { "source": 4, - "target": 87, + "target": 88, "type": "produces" }, { - "source": 88, + "source": 89, "target": 15, "type": "consumes" }, { - "source": 88, + "source": 89, "target": 3, "type": "consumes" }, { - "source": 88, + "source": 89, "target": 19, "type": "consumes" }, { "source": 2, - "target": 88, + "target": 89, "type": "produces" }, { "source": 3, - "target": 88, + "target": 89, "type": "produces" }, { - "source": 89, + "source": 90, "target": 68, "type": "consumes" }, { "source": 4, - "target": 89, + "target": 90, "type": "produces" }, { - "source": 90, + "source": 91, "target": 7, "type": "consumes" }, { "source": 7, - "target": 90, + "target": 91, "type": "produces" }, { "source": 46, - "target": 90, + "target": 91, "type": "produces" }, { "source": 19, - "target": 90, + "target": 91, "type": "produces" }, { - "source": 91, + "source": 92, "target": 3, "type": "consumes" }, { - "source": 71, - "target": 91, + "source": 72, + "target": 92, "type": "produces" }, { - "source": 92, + "source": 93, "target": 7, "type": "consumes" }, { - "source": 92, + "source": 93, "target": 12, "type": "consumes" }, { "source": 7, - "target": 92, + "target": 93, "type": "produces" }, { "source": 4, - "target": 92, + "target": 93, "type": "produces" }, { "source": 15, - "target": 92, + "target": 93, "type": "produces" }, { "source": 16, - "target": 92, + "target": 93, "type": "produces" }, { "source": 5, - "target": 92, + "target": 93, "type": "produces" }, { - "source": 93, + "source": 94, "target": 12, "type": "consumes" }, { - "source": 94, - "target": 93, + "source": 95, + "target": 94, "type": "produces" }, { - "source": 95, + "source": 96, "target": 12, "type": "consumes" }, { "source": 12, - "target": 95, + "target": 96, "type": "produces" }, { - "source": 96, + "source": 97, "target": 12, "type": "consumes" }, { - "source": 94, - "target": 96, + "source": 95, + "target": 97, "type": "produces" }, { - "source": 97, + "source": 98, "target": 7, "type": "consumes" }, { "source": 7, - "target": 97, + "target": 98, "type": "produces" }, { - "source": 98, + "source": 99, "target": 7, "type": "consumes" }, { "source": 7, - "target": 98, + "target": 99, "type": "produces" }, { - "source": 99, + "source": 100, "target": 2, "type": "consumes" }, { "source": 4, - "target": 99, + "target": 100, "type": "produces" }, { - "source": 100, + "source": 101, "target": 2, "type": "consumes" }, { - "source": 100, + "source": 101, "target": 3, "type": "consumes" }, { "source": 7, - "target": 100, + "target": 101, "type": "produces" }, { "source": 4, - "target": 100, + "target": 101, "type": "produces" }, { - "source": 101, + "source": 102, "target": 3, "type": "consumes" }, { "source": 4, - "target": 101, + "target": 102, "type": "produces" }, { "source": 16, - "target": 101, + "target": 102, "type": "produces" }, { "source": 5, - "target": 101, + "target": 102, "type": "produces" }, { - "source": 102, + "source": 103, "target": 7, "type": "consumes" }, { - "source": 102, + "source": 103, "target": 19, "type": "consumes" }, { "source": 7, - "target": 102, + "target": 103, "type": "produces" }, { - "source": 103, + "source": 104, "target": 7, "type": "consumes" }, { "source": 7, - "target": 103, + "target": 104, "type": "produces" }, { - "source": 104, + "source": 105, "target": 2, "type": "consumes" }, { - "source": 104, + "source": 105, "target": 68, "type": "consumes" }, { "source": 4, - "target": 104, + "target": 105, "type": "produces" }, { "source": 68, - "target": 104, + "target": 105, "type": "produces" }, { - "source": 105, + "source": 106, "target": 2, "type": "consumes" }, { - "source": 105, + "source": 106, "target": 68, "type": "consumes" }, { "source": 4, - "target": 105, + "target": 106, "type": "produces" }, { "source": 68, - "target": 105, + "target": 106, "type": "produces" }, { - "source": 106, + "source": 107, "target": 2, "type": "consumes" }, { - "source": 106, + "source": 107, "target": 68, "type": "consumes" }, { "source": 68, - "target": 106, + "target": 107, "type": "produces" }, { - "source": 107, + "source": 108, "target": 7, "type": "consumes" }, { "source": 7, - "target": 107, + "target": 108, "type": "produces" }, { - "source": 108, + "source": 109, "target": 7, "type": "consumes" }, { "source": 46, - "target": 108, + "target": 109, "type": "produces" }, { - "source": 109, + "source": 110, "target": 7, "type": "consumes" }, { - "source": 109, + "source": 110, "target": 12, "type": "consumes" }, { - "source": 109, - "target": 110, + "source": 110, + "target": 111, "type": "consumes" }, { "source": 15, - "target": 109, + "target": 110, "type": "produces" }, { - "source": 111, + "source": 112, "target": 61, "type": "consumes" }, { - "source": 111, + "source": 112, "target": 62, "type": "consumes" }, { "source": 43, - "target": 111, + "target": 112, "type": "produces" }, { - "source": 112, + "source": 113, "target": 43, "type": "consumes" }, { "source": 10, - "target": 112, + "target": 113, "type": "produces" }, { - "source": 113, + "source": 114, "target": 7, "type": "consumes" }, { "source": 7, - "target": 113, + "target": 114, "type": "produces" }, { - "source": 114, + "source": 115, "target": 3, "type": "consumes" }, { "source": 19, - "target": 114, + "target": 115, "type": "produces" }, { - "source": 115, + "source": 116, "target": 2, "type": "consumes" }, { "source": 4, - "target": 115, + "target": 116, "type": "produces" }, { - "source": 116, + "source": 117, "target": 7, "type": "consumes" }, { "source": 7, - "target": 116, + "target": 117, "type": "produces" }, { - "source": 117, + "source": 118, "target": 7, "type": "consumes" }, { "source": 46, - "target": 117, + "target": 118, "type": "produces" }, { "source": 19, - "target": 117, + "target": 118, "type": "produces" }, { - "source": 118, + "source": 119, "target": 7, "type": "consumes" }, { "source": 7, - "target": 118, + "target": 119, "type": "produces" }, { - "source": 119, + "source": 120, "target": 7, "type": "consumes" }, { "source": 7, - "target": 119, + "target": 120, "type": "produces" }, { - "source": 120, + "source": 121, "target": 7, "type": "consumes" }, { "source": 46, - "target": 120, + "target": 121, "type": "produces" }, { - "source": 121, + "source": 122, "target": 3, "type": "consumes" }, { "source": 4, - "target": 121, + "target": 122, "type": "produces" }, { - "source": 122, + "source": 123, "target": 19, "type": "consumes" }, { "source": 62, - "target": 122, + "target": 123, "type": "produces" }, { - "source": 123, - "target": 124, + "source": 124, + "target": 125, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 7, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 22, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 2, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 12, "type": "consumes" }, { - "source": 123, - "target": 110, + "source": 124, + "target": 111, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 62, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 24, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 3, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 19, "type": "consumes" }, { - "source": 123, + "source": 124, "target": 49, "type": "consumes" }, { "source": 7, - "target": 123, + "target": 124, "type": "produces" }, { "source": 4, - "target": 123, + "target": 124, "type": "produces" }, { "source": 12, - "target": 123, + "target": 124, "type": "produces" }, { "source": 15, - "target": 123, + "target": 124, "type": "produces" }, { "source": 61, - "target": 123, + "target": 124, "type": "produces" }, { - "source": 125, + "source": 126, "target": 15, "type": "consumes" }, { "source": 7, - "target": 125, + "target": 126, "type": "produces" }, { "source": 46, - "target": 125, + "target": 126, "type": "produces" }, { - "source": 126, + "source": 127, "target": 7, "type": "consumes" }, { "source": 7, - "target": 126, + "target": 127, "type": "produces" }, { - "source": 127, + "source": 128, "target": 7, "type": "consumes" }, { "source": 7, - "target": 127, + "target": 128, "type": "produces" }, { - "source": 128, + "source": 129, "target": 7, "type": "consumes" }, { - "source": 128, + "source": 129, "target": 22, "type": "consumes" }, { - "source": 129, + "source": 130, "target": 2, "type": "consumes" }, { - "source": 129, + "source": 130, "target": 3, "type": "consumes" }, { "source": 4, - "target": 129, + "target": 130, "type": "produces" }, { "source": 5, - "target": 129, + "target": 130, "type": "produces" }, { - "source": 130, + "source": 131, "target": 7, "type": "consumes" }, { "source": 7, - "target": 130, + "target": 131, "type": "produces" }, { - "source": 131, + "source": 132, "target": 43, "type": "consumes" }, { - "source": 131, + "source": 132, "target": 10, "type": "consumes" }, { "source": 4, - "target": 131, + "target": 132, "type": "produces" }, { "source": 5, - "target": 131, - "type": "produces" - }, - { - "source": 132, - "target": 10, - "type": "consumes" - }, - { - "source": 67, "target": 132, "type": "produces" }, diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 54249c22e..d54577a7e 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -76,6 +76,7 @@ | docker_pull | scan | No | Download images from a docker repository | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-24 | | dockerhub | scan | No | Search for docker repositories of discovered orgs/usernames | code-enum, passive, safe | ORG_STUB, SOCIAL | CODE_REPOSITORY, SOCIAL, URL_UNVERIFIED | @domwhewell-sage | 2024-03-12 | | emailformat | scan | No | Query email-format.com for email addresses | email-enum, passive, safe | DNS_NAME | EMAIL_ADDRESS | @TheTechromancer | 2022-07-11 | +| extractous | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 | | fullhunt | scan | Yes | Query the fullhunt.io API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-24 | | git_clone | scan | No | Clone code github repositories | code-enum, passive, safe, slow | CODE_REPOSITORY | FILESYSTEM | @domwhewell-sage | 2024-03-08 | | github_codesearch | scan | Yes | Query Github's API for code containing the target domain name | code-enum, passive, safe, subdomain-enum | DNS_NAME | CODE_REPOSITORY, URL_UNVERIFIED | @domwhewell-sage | 2023-12-14 | @@ -105,7 +106,6 @@ | subdomainradar | scan | Yes | Query the Subdomain API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-08 | | trickest | scan | Yes | Query Trickest's API for subdomains | affiliates, passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @amiremami | 2024-07-27 | | trufflehog | scan | No | TruffleHog is a tool for finding credentials | code-enum, passive, safe | CODE_REPOSITORY, FILESYSTEM | FINDING, VULNERABILITY | @domwhewell-sage | 2024-03-12 | -| unstructured | scan | No | Module to extract data from files | passive, safe | FILESYSTEM | RAW_TEXT | @domwhewell-sage | 2024-06-03 | | urlscan | scan | No | Query urlscan.io for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME, URL_UNVERIFIED | @TheTechromancer | 2022-06-09 | | viewdns | scan | No | Query viewdns.info's reverse whois for related domains | affiliates, passive, safe | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-07-04 | | virustotal | scan | Yes | Query VirusTotal's API for subdomains | passive, safe, subdomain-enum | DNS_NAME | DNS_NAME | @TheTechromancer | 2022-08-25 | @@ -121,6 +121,7 @@ | python | output | No | Output via Python API | | * | | @TheTechromancer | 2022-09-13 | | slack | output | No | Message a Slack channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | | splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | @w0Tx | 2024-02-17 | +| sqlite | output | No | Output scan data to a SQLite database | | * | | | | | stdout | output | No | Output to text | | * | | | | | subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | @TheTechromancer | 2023-07-31 | | teams | output | No | Message a Teams channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index 6b030cb9d..97b3692c2 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -69,14 +69,14 @@ Presets: Modules: -m MODULE [MODULE ...], --modules MODULE [MODULE ...] - Modules to enable. Choices: emailformat,ntlm,hunt,bucket_digitalocean,bucket_google,columbus,portscan,ffuf,paramminer_getparams,wappalyzer,affiliates,censys,azure_tenant,dockerhub,bucket_amazon,ipneighbor,anubisdb,dastardly,viewdns,builtwith,trufflehog,bevigil,dnscaa,baddns_direct,vhost,docker_pull,ip2location,shodan_dns,wafw00f,bucket_firebase,digitorus,trickest,hackertarget,subdomainradar,ffuf_shortnames,dotnetnuke,smuggler,iis_shortnames,gowitness,dnsbrute_mutations,dnsbrute,paramminer_cookies,fingerprintx,baddns_zone,hunterio,github_workflows,github_org,httpx,bufferoverrun,wpscan,securitytxt,sslcert,wayback,asn,github_codesearch,nuclei,virustotal,dnscommonsrv,otx,credshed,bucket_azure,ipstack,code_repository,sitedossier,git_clone,skymem,google_playstore,zoomeye,url_manipulation,apkpure,bypass403,internetdb,leakix,paramminer_headers,urlscan,gitlab,git,baddns,rapiddns,subdomaincenter,oauth,azure_realm,chaos,myssl,host_header,securitytrails,passivetotal,filedownload,postman_download,c99,telerik,fullhunt,crt,secretsdb,binaryedge,ajaxpro,certspotter,dnsdumpster,generic_ssrf,bucket_file_enum,dehashed,unstructured,postman,pgp,badsecrets,social,newsletters,robots + Modules to enable. Choices: wpscan,crt,c99,credshed,gitlab,ajaxpro,pgp,trickest,ntlm,robots,postman_download,skymem,otx,rapiddns,bucket_firebase,shodan_dns,fingerprintx,internetdb,ip2location,columbus,bucket_azure,censys,binaryedge,github_org,wappalyzer,nuclei,ipneighbor,baddns_zone,telerik,url_manipulation,zoomeye,virustotal,myssl,builtwith,dnscaa,emailformat,oauth,bucket_amazon,sslcert,git,iis_shortnames,bucket_google,filedownload,bevigil,social,paramminer_headers,paramminer_cookies,digitorus,wafw00f,portscan,bufferoverrun,github_codesearch,gowitness,google_playstore,ipstack,code_repository,github_workflows,baddns_direct,secretsdb,chaos,azure_realm,affiliates,viewdns,dockerhub,subdomainradar,hunterio,dnscommonsrv,ffuf_shortnames,trufflehog,dotnetnuke,hunt,baddns,host_header,dastardly,vhost,docker_pull,paramminer_getparams,dnsdumpster,postman,httpx,bucket_digitalocean,bypass403,leakix,securitytrails,extractous,dnsbrute,ffuf,smuggler,asn,passivetotal,subdomaincenter,dnsbrute_mutations,wayback,newsletters,hackertarget,sitedossier,securitytxt,generic_ssrf,urlscan,badsecrets,dehashed,certspotter,fullhunt,anubisdb,apkpure,azure_tenant,git_clone,bucket_file_enum -l, --list-modules List available modules. -lmo, --list-module-options Show all module config options -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...] Exclude these modules. -f FLAG [FLAG ...], --flags FLAG [FLAG ...] - Enable modules by flag. Choices: email-enum,affiliates,web-screenshots,code-enum,passive,web-thorough,portscan,baddns,subdomain-enum,safe,service-enum,deadly,aggressive,iis-shortnames,slow,active,report,subdomain-hijack,social-enum,web-basic,web-paramminer,cloud-enum + Enable modules by flag. Choices: web-screenshots,service-enum,deadly,portscan,code-enum,iis-shortnames,cloud-enum,web-thorough,safe,report,active,affiliates,web-paramminer,aggressive,baddns,subdomain-hijack,social-enum,subdomain-enum,email-enum,web-basic,slow,passive -lf, --list-flags List available flags. -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...] Only enable modules with these flags (e.g. -rf passive) @@ -101,7 +101,7 @@ Output: -o DIR, --output-dir DIR Directory to output scan results -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] - Output module(s). Choices: slack,emails,splunk,web_report,txt,json,discord,stdout,subdomains,http,neo4j,csv,websocket,python,teams,asset_inventory + Output module(s). Choices: websocket,neo4j,discord,splunk,json,csv,sqlite,http,python,subdomains,stdout,teams,asset_inventory,slack,web_report,txt,emails --json, -j Output scan data in JSON format --brief, -br Output only the data itself --event-types EVENT_TYPES [EVENT_TYPES ...] diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index ae1ef80ca..1ab2dea87 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -393,6 +393,7 @@ Many modules accept their own configuration options. These options have the abil | modules.dnscaa.urls | bool | emit URL_UNVERIFIED events | True | | modules.docker_pull.all_tags | bool | Download all tags from each registry (Default False) | False | | modules.docker_pull.output_folder | str | Folder to download docker repositories to | | +| modules.extractous.extensions | list | File extensions to parse | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | | modules.fullhunt.api_key | str | FullHunt API Key | | | modules.git_clone.api_key | str | Github token | | | modules.git_clone.output_folder | str | Folder to clone repositories to | | @@ -425,8 +426,7 @@ Many modules accept their own configuration options. These options have the abil | modules.trufflehog.config | str | File path or URL to YAML trufflehog config | | | modules.trufflehog.deleted_forks | bool | Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours. | False | | modules.trufflehog.only_verified | bool | Only report credentials that have been verified | True | -| modules.trufflehog.version | str | trufflehog version | 3.83.3 | -| modules.unstructured.extensions | list | File extensions to parse | ['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'] | +| modules.trufflehog.version | str | trufflehog version | 3.83.4 | | modules.urlscan.urls | bool | Emit URLs in addition to DNS_NAMEs | False | | modules.virustotal.api_key | str | VirusTotal API Key | | | modules.wayback.garbage_threshold | int | Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data) | 10 | @@ -463,6 +463,7 @@ Many modules accept their own configuration options. These options have the abil | modules.splunk.source | str | Source path to be added to the metadata | | | modules.splunk.timeout | int | HTTP timeout | 10 | | modules.splunk.url | str | Web URL | | +| modules.sqlite.database | str | The path to the sqlite database file | | | modules.stdout.accept_dupes | bool | Whether to show duplicate events, default True | True | | modules.stdout.event_fields | list | Which event fields to display | [] | | modules.stdout.event_types | list | Which events to display, default all event types | [] | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index 691b7303a..52ef9065d 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -106,14 +106,14 @@ Below is a full list of event types along with which modules produce/consume the | Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | |---------------------|-----------------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| * | 15 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, neo4j, python, slack, splunk, stdout, teams, txt, websocket | | +| * | 16 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, neo4j, python, slack, splunk, sqlite, stdout, teams, txt, websocket | | | ASN | 0 | 1 | | asn | | AZURE_TENANT | 1 | 0 | speculate | | | CODE_REPOSITORY | 6 | 6 | docker_pull, git_clone, github_workflows, google_playstore, postman_download, trufflehog | code_repository, dockerhub, github_codesearch, github_org, gitlab, postman | | DNS_NAME | 58 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crt, dehashed, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, myssl, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | | DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | | EMAIL_ADDRESS | 1 | 9 | emails | credshed, dehashed, dnscaa, emailformat, hunterio, pgp, securitytxt, skymem, sslcert | -| FILESYSTEM | 2 | 6 | trufflehog, unstructured | apkpure, docker_pull, filedownload, git_clone, github_workflows, postman_download | +| FILESYSTEM | 2 | 6 | extractous, trufflehog | apkpure, docker_pull, filedownload, git_clone, github_workflows, postman_download | | FINDING | 2 | 29 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | | GEOLOCATION | 0 | 2 | | ip2location, ipstack | | HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | @@ -125,7 +125,7 @@ Below is a full list of event types along with which modules produce/consume the | ORG_STUB | 4 | 1 | dockerhub, github_org, google_playstore, postman | speculate | | PASSWORD | 0 | 2 | | credshed, dehashed | | PROTOCOL | 0 | 1 | | fingerprintx | -| RAW_TEXT | 1 | 1 | excavate | unstructured | +| RAW_TEXT | 1 | 1 | excavate | extractous | | SOCIAL | 6 | 3 | dockerhub, github_org, gitlab, gowitness, postman, speculate | dockerhub, gitlab, social | | STORAGE_BUCKET | 8 | 5 | baddns_direct, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, speculate | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google | | TECHNOLOGY | 4 | 8 | asset_inventory, gitlab, web_report, wpscan | badsecrets, dotnetnuke, gitlab, gowitness, internetdb, nuclei, wappalyzer, wpscan | diff --git a/docs/scanning/index.md b/docs/scanning/index.md index e2e4a7921..55ac36e59 100644 --- a/docs/scanning/index.md +++ b/docs/scanning/index.md @@ -107,30 +107,30 @@ A single module can have multiple flags. For example, the `securitytrails` modul ### List of Flags -| Flag | # Modules | Description | Modules | -|------------------|-------------|----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| safe | 89 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, apkpure, asn, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bufferoverrun, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, google_playstore, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portscan, postman, postman_download, rapiddns, robots, secretsdb, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, subdomainradar, trickest, trufflehog, unstructured, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | -| passive | 64 | Never connects to target systems | affiliates, aggregate, anubisdb, apkpure, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, bufferoverrun, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, fullhunt, git_clone, github_codesearch, github_org, github_workflows, google_playstore, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, myssl, otx, passivetotal, pgp, postman, postman_download, rapiddns, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, subdomainradar, trickest, trufflehog, unstructured, urlscan, viewdns, virustotal, wayback, zoomeye | -| subdomain-enum | 51 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_direct, baddns_zone, bevigil, binaryedge, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, virustotal, wayback, zoomeye | -| active | 47 | Makes active connections to target systems | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dnsbrute, dnsbrute_mutations, dnscommonsrv, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, newsletters, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, portscan, robots, secretsdb, securitytxt, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer, wpscan | -| aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dnsbrute, dnsbrute_mutations, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f, wpscan | -| web-basic | 18 | Basic, non-intrusive web scan functionality | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, securitytxt, sslcert, wappalyzer | -| cloud-enum | 14 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth, securitytxt | -| code-enum | 14 | Find public code repositories and search them for secrets etc. | apkpure, code_repository, docker_pull, dockerhub, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, google_playstore, postman, postman_download, trufflehog | -| web-thorough | 12 | More advanced web scanning functionality | ajaxpro, bucket_digitalocean, bypass403, dastardly, dotnetnuke, ffuf_shortnames, generic_ssrf, host_header, hunt, smuggler, telerik, url_manipulation | -| slow | 11 | May take a long time to complete | bucket_digitalocean, dastardly, dnsbrute_mutations, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | -| affiliates | 9 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, trickest, viewdns, zoomeye | -| email-enum | 8 | Enumerates email addresses | dehashed, dnscaa, emailformat, emails, hunterio, pgp, skymem, sslcert | -| deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | -| baddns | 3 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_direct, baddns_zone | -| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | -| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | -| portscan | 2 | Discovers open ports | internetdb, portscan | -| report | 2 | Generates a report at the end of the scan | affiliates, asn | -| social-enum | 2 | Enumerates social media | httpx, social | -| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | -| subdomain-hijack | 1 | Detects hijackable subdomains | baddns | -| web-screenshots | 1 | Takes screenshots of web pages | gowitness | +| Flag | # Modules | Description | Modules | +|------------------|-------------|----------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| safe | 89 | Non-intrusive, safe to run | affiliates, aggregate, ajaxpro, anubisdb, apkpure, asn, azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, badsecrets, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, bufferoverrun, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnscommonsrv, dnsdumpster, docker_pull, dockerhub, emailformat, extractous, filedownload, fingerprintx, fullhunt, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, google_playstore, gowitness, hackertarget, httpx, hunt, hunterio, iis_shortnames, internetdb, ip2location, ipstack, leakix, myssl, newsletters, ntlm, oauth, otx, passivetotal, pgp, portscan, postman, postman_download, rapiddns, robots, secretsdb, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, social, sslcert, subdomaincenter, subdomainradar, trickest, trufflehog, urlscan, viewdns, virustotal, wappalyzer, wayback, zoomeye | +| passive | 64 | Never connects to target systems | affiliates, aggregate, anubisdb, apkpure, asn, azure_realm, azure_tenant, bevigil, binaryedge, bucket_file_enum, bufferoverrun, builtwith, c99, censys, certspotter, chaos, code_repository, columbus, credshed, crt, dehashed, digitorus, dnscaa, dnsdumpster, docker_pull, dockerhub, emailformat, excavate, extractous, fullhunt, git_clone, github_codesearch, github_org, github_workflows, google_playstore, hackertarget, hunterio, internetdb, ip2location, ipneighbor, ipstack, leakix, myssl, otx, passivetotal, pgp, postman, postman_download, rapiddns, securitytrails, shodan_dns, sitedossier, skymem, social, speculate, subdomaincenter, subdomainradar, trickest, trufflehog, urlscan, viewdns, virustotal, wayback, zoomeye | +| subdomain-enum | 51 | Enumerates subdomains | anubisdb, asn, azure_realm, azure_tenant, baddns_direct, baddns_zone, bevigil, binaryedge, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, github_codesearch, github_org, hackertarget, httpx, hunterio, internetdb, ipneighbor, leakix, myssl, oauth, otx, passivetotal, postman, postman_download, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, sslcert, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, virustotal, wayback, zoomeye | +| active | 47 | Makes active connections to target systems | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, dnsbrute, dnsbrute_mutations, dnscommonsrv, dotnetnuke, ffuf, ffuf_shortnames, filedownload, fingerprintx, generic_ssrf, git, gitlab, gowitness, host_header, httpx, hunt, iis_shortnames, newsletters, ntlm, nuclei, oauth, paramminer_cookies, paramminer_getparams, paramminer_headers, portscan, robots, secretsdb, securitytxt, smuggler, sslcert, telerik, url_manipulation, vhost, wafw00f, wappalyzer, wpscan | +| aggressive | 20 | Generates a large amount of network traffic | bypass403, dastardly, dnsbrute, dnsbrute_mutations, dotnetnuke, ffuf, ffuf_shortnames, generic_ssrf, host_header, ipneighbor, nuclei, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, telerik, url_manipulation, vhost, wafw00f, wpscan | +| web-basic | 18 | Basic, non-intrusive web scan functionality | azure_realm, baddns, badsecrets, bucket_amazon, bucket_azure, bucket_firebase, bucket_google, filedownload, git, httpx, iis_shortnames, ntlm, oauth, robots, secretsdb, securitytxt, sslcert, wappalyzer | +| cloud-enum | 14 | Enumerates cloud resources | azure_realm, azure_tenant, baddns, baddns_direct, baddns_zone, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, httpx, oauth, securitytxt | +| code-enum | 14 | Find public code repositories and search them for secrets etc. | apkpure, code_repository, docker_pull, dockerhub, git, git_clone, github_codesearch, github_org, github_workflows, gitlab, google_playstore, postman, postman_download, trufflehog | +| web-thorough | 12 | More advanced web scanning functionality | ajaxpro, bucket_digitalocean, bypass403, dastardly, dotnetnuke, ffuf_shortnames, generic_ssrf, host_header, hunt, smuggler, telerik, url_manipulation | +| slow | 11 | May take a long time to complete | bucket_digitalocean, dastardly, dnsbrute_mutations, docker_pull, fingerprintx, git_clone, paramminer_cookies, paramminer_getparams, paramminer_headers, smuggler, vhost | +| affiliates | 9 | Discovers affiliated hostnames/domains | affiliates, azure_realm, azure_tenant, builtwith, oauth, sslcert, trickest, viewdns, zoomeye | +| email-enum | 8 | Enumerates email addresses | dehashed, dnscaa, emailformat, emails, hunterio, pgp, skymem, sslcert | +| deadly | 4 | Highly aggressive | dastardly, ffuf, nuclei, vhost | +| baddns | 3 | Runs all modules from the DNS auditing tool BadDNS | baddns, baddns_direct, baddns_zone | +| web-paramminer | 3 | Discovers HTTP parameters through brute-force | paramminer_cookies, paramminer_getparams, paramminer_headers | +| iis-shortnames | 2 | Scans for IIS Shortname vulnerability | ffuf_shortnames, iis_shortnames | +| portscan | 2 | Discovers open ports | internetdb, portscan | +| report | 2 | Generates a report at the end of the scan | affiliates, asn | +| social-enum | 2 | Enumerates social media | httpx, social | +| service-enum | 1 | Identifies protocols running on open ports | fingerprintx | +| subdomain-hijack | 1 | Detects hijackable subdomains | baddns | +| web-screenshots | 1 | Takes screenshots of web pages | gowitness | ## Dependencies From 7a42e11b0a394b3317f9d27f21df27a2f14375c2 Mon Sep 17 00:00:00 2001 From: blsaccess Date: Mon, 11 Nov 2024 00:23:13 +0000 Subject: [PATCH 07/10] Update trufflehog --- bbot/modules/trufflehog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index d5bdb4cfa..4f0d91686 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -13,7 +13,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.83.4", + "version": "3.83.6", "config": "", "only_verified": True, "concurrency": 8, From 454235ea063ca7480310fde50fb37bb91aaf9358 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 04:30:09 +0000 Subject: [PATCH 08/10] Bump werkzeug from 3.1.1 to 3.1.3 Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.1.1 to 3.1.3. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/3.1.1...3.1.3) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index fb5a3e77e..d2447585d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2929,13 +2929,13 @@ files = [ [[package]] name = "werkzeug" -version = "3.1.1" +version = "3.1.3" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" files = [ - {file = "werkzeug-3.1.1-py3-none-any.whl", hash = "sha256:a71124d1ef06008baafa3d266c02f56e1836a5984afd6dd6c9230669d60d9fb5"}, - {file = "werkzeug-3.1.1.tar.gz", hash = "sha256:8cd39dfbdfc1e051965f156163e2974e52c210f130810e9ad36858f0fd3edad4"}, + {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, + {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, ] [package.dependencies] From 91eab7671a49b59096ca85361f984df9cebedafd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 04:30:31 +0000 Subject: [PATCH 09/10] Bump tldextract from 5.1.2 to 5.1.3 Bumps [tldextract](https://github.com/john-kurkowski/tldextract) from 5.1.2 to 5.1.3. - [Release notes](https://github.com/john-kurkowski/tldextract/releases) - [Changelog](https://github.com/john-kurkowski/tldextract/blob/master/CHANGELOG.md) - [Commits](https://github.com/john-kurkowski/tldextract/compare/5.1.2...5.1.3) --- updated-dependencies: - dependency-name: tldextract dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index fb5a3e77e..a0ed91b36 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2670,13 +2670,13 @@ widechars = ["wcwidth"] [[package]] name = "tldextract" -version = "5.1.2" +version = "5.1.3" description = "Accurately separates a URL's subdomain, domain, and public suffix, using the Public Suffix List (PSL). By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "tldextract-5.1.2-py3-none-any.whl", hash = "sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46"}, - {file = "tldextract-5.1.2.tar.gz", hash = "sha256:c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200"}, + {file = "tldextract-5.1.3-py3-none-any.whl", hash = "sha256:78de310cc2ca018692de5ddf320f9d6bd7c5cf857d0fd4f2175f0cdf4440ea75"}, + {file = "tldextract-5.1.3.tar.gz", hash = "sha256:d43c7284c23f5dc8a42fd0fee2abede2ff74cc622674e4cb07f514ab3330c338"}, ] [package.dependencies] @@ -2687,7 +2687,7 @@ requests-file = ">=1.4" [package.extras] release = ["build", "twine"] -testing = ["black", "mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "types-filelock", "types-requests"] +testing = ["mypy", "pytest", "pytest-gitignore", "pytest-mock", "responses", "ruff", "syrupy", "tox", "tox-uv", "types-filelock", "types-requests"] [[package]] name = "tomli" From 58624cfef16d4cddc985091e1ec79ac95f337e7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 04:30:53 +0000 Subject: [PATCH 10/10] Bump mkdocs-material from 9.5.43 to 9.5.44 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.5.43 to 9.5.44. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.5.43...9.5.44) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index fb5a3e77e..b4ae403bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1278,13 +1278,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.43" +version = "9.5.44" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.43-py3-none-any.whl", hash = "sha256:4aae0664c456fd12837a3192e0225c17960ba8bf55d7f0a7daef7e4b0b914a34"}, - {file = "mkdocs_material-9.5.43.tar.gz", hash = "sha256:83be7ff30b65a1e4930dfa4ab911e75780a3afc9583d162692e434581cb46979"}, + {file = "mkdocs_material-9.5.44-py3-none-any.whl", hash = "sha256:47015f9c167d58a5ff5e682da37441fc4d66a1c79334bfc08d774763cacf69ca"}, + {file = "mkdocs_material-9.5.44.tar.gz", hash = "sha256:f3a6c968e524166b3f3ed1fb97d3ed3e0091183b0545cedf7156a2a6804c56c0"}, ] [package.dependencies]