Dev --> Stable 1.1.1

.github/workflows/tests.yml

@@ -5,9 +5,6 @@ on:
       - stable
       - dev
   pull_request:
-  pull_request_target:
-    types:
-      - closed
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
@@ -46,7 +43,7 @@ jobs:
           poetry install
       - name: Run tests
         run: |
-          poetry run pytest --exitfirst --reruns 2 --disable-warnings --log-cli-level=DEBUG --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 600 --disable-warnings --log-cli-level=DEBUG --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v3
         with:
@@ -56,6 +53,7 @@ jobs:
   update_docs:
     needs: test
     runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     steps:
       - uses: actions/checkout@v3
         with:
@@ -82,7 +80,7 @@ jobs:
   publish_docs:
     needs: update_docs
     runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.event.pull_request.base.ref == 'dev'
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev')
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -95,12 +93,15 @@ jobs:
           path: .cache
           restore-keys: |
             mkdocs-material-
-      - run: pip install mkdocs-material mkdocs-extra-sass-plugin livereload
-      - run: mkdocs gh-deploy --force
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --only=docs
+      - run: poetry run mkdocs gh-deploy --force
   publish_code:
     needs: update_docs
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request_target' && github.event.pull_request.merged == true && (github.event.pull_request.base.ref == 'dev' || github.event.pull_request.base.ref == 'stable')
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true
     steps:
       - uses: actions/checkout@v3
@@ -128,23 +129,23 @@ jobs:
         run: |
           echo "::set-output name=BBOT_VERSION::$(poetry version | cut -d' ' -f2 | tr -d v)"
       - name: Publish to Docker Hub (dev)
-        if: github.ref == 'refs/heads/dev'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
         uses: elgohr/Publish-Docker-Github-Action@v5
         with:
           name: blacklanternsecurity/bbot
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           tags: "latest,dev,${{ steps.version.outputs.BBOT_VERSION }}"
       - name: Publish to Docker Hub (stable)
-        if: github.ref == 'refs/heads/stable'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
         uses: elgohr/Publish-Docker-Github-Action@v5
         with:
           name: blacklanternsecurity/bbot
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
           tags: "stable,${{ steps.version.outputs.BBOT_VERSION }}"
       - name: Docker Hub Description
-        if: github.ref == 'refs/heads/dev'
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
         uses: peter-evans/dockerhub-description@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}

README.md

@@ -4,7 +4,7 @@
 
 ### OSINT automation for hackers.
 
-[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Pypi Downloads](https://img.shields.io/pypi/dm/bbot)](https://pypi.org/project/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
+[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Demo Labs 2023](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://forum.defcon.org/node/246338) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Pypi Downloads](https://img.shields.io/pypi/dm/bbot)](https://pypistats.org/packages/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
 
 BBOT (Bighuge BLS OSINT Tool) is a modular, recursive OSINT framework that can execute the entire OSINT workflow in a single command.
 
@@ -18,13 +18,9 @@ BBOT typically outperforms other subdomain enumeration tools by 20-25%. To learn
 
 ## Installation ([pip](https://pypi.org/project/bbot/))
 
-For more installation methods including [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot), see [Installation](https://www.blacklanternsecurity.com/bbot/#installation).
+Note: Requires Linux and Python 3.9+. For more installation methods including [Docker](https://hub.docker.com/r/blacklanternsecurity/bbot), see [Installation](https://www.blacklanternsecurity.com/bbot/#installation).
 
 ```bash
-# Prerequisites:
-# - Linux (Windows and macOS are *not* supported)
-# - Python 3.9 or newer
-
 # stable version
 pipx install bbot
 
@@ -84,7 +80,7 @@ bbot -t evilcorp.com -f subdomain-enum email-enum cloud-enum web-basic -m nmap g
 
 ## Targets
 
-BBOT accepts an unlimited number of targets. You can specify targets either directly on the command line or in files (or both!). Targets can be any of the following:
+BBOT accepts an unlimited number of targets via `-t`. You can specify targets either directly on the command line or in files (or both!). Targets can be any of the following:
 
 - `DNS_NAME` (`evilcorp.com`)
 - `IP_ADDRESS` (`1.2.3.4`)
@@ -94,8 +90,34 @@ BBOT accepts an unlimited number of targets. You can specify targets either dire
 
 For more information, see [Targets](https://www.blacklanternsecurity.com/bbot/scanning/#targets-t). To learn how BBOT handles scope, see [Scope](https://www.blacklanternsecurity.com/bbot/scanning/#scope).
 
+## API Keys
+
+Similar to Amass or Subfinder, BBOT supports API keys for various third-party services such as SecurityTrails, etc.
+
+The standard way to do this is to enter your API keys in **`~/.config/bbot/secrets.yml`**:
+```yaml
+modules:
+  shodan_dns:
+    api_key: 4f41243847da693a4f356c0486114bc6
+  c99:
+    api_key: 21a270d5f59c9b05813a72bb41707266
+  virustotal:
+    api_key: dd5f0eee2e4a99b71a939bded450b246
+  securitytrails:
+    api_key: d9a05c3fd9a514497713c54b4455d0b0
+```
+
+If you like, you can also specify them on the command line:
+```bash
+bbot -c modules.virustotal.api_key=dd5f0eee2e4a99b71a939bded450b246
+```
+
+For details, see [Configuration](https://www.blacklanternsecurity.com/bbot/scanning/configuration/)
+
 ## BBOT as a Python library
 
+BBOT exposes a Python API that allows it to be used for all kinds of fun and nefarious purposes, like a [Discord Bot that responds to `/scan evilcorp.com`](https://www.blacklanternsecurity.com/bbot/dev/#bbot-python-library-advanced-usage#discord-bot-example).
+
 **Synchronous**
 
 ```python

bbot/agent/agent.py

@@ -49,9 +49,17 @@ async def ws(self, rebuild=False):
             verbs = ("Building", "Built")
             if rebuild:
                 verbs = ("Rebuilding", "Rebuilt")
-            log.debug(f"{verbs[0]} websocket connection to {self.url}")
-            self._ws = await websockets.connect(self.url, **kwargs)
-            log.debug(f"{verbs[1]} websocket connection to {self.url}")
+            url = f"{self.url}/control/"
+            log.debug(f"{verbs[0]} websocket connection to {url}")
+            while 1:
+                try:
+                    self._ws = await websockets.connect(url, **kwargs)
+                    break
+                except Exception as e:
+                    log.error(f'Failed to establish websockets connection to URL "{url}": {e}')
+                    log.trace(traceback.format_exc())
+                    await asyncio.sleep(1)
+            log.debug(f"{verbs[1]} websocket connection to {url}")
         return self._ws
 
     async def start(self):
@@ -69,6 +77,7 @@ async def start(self):
                     if message.command == "ping":
                         if self.scan is None:
                             await self.send({"conversation": str(message.conversation), "message_type": "pong"})
+                            continue
 
                     command_type = getattr(messages, message.command, None)
                     if command_type is None:

bbot/cli.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import os
+import re
 import sys
 import asyncio
 import logging
@@ -138,6 +139,7 @@ async def _main():
                     *options.targets,
                     modules=list(modules),
                     output_modules=list(output_modules),
+                    output_dir=options.output_dir,
                     config=config,
                     name=options.name,
                     whitelist=options.whitelist,
@@ -297,7 +299,7 @@ async def _main():
 
                 scanner.helpers.word_cloud.load()
 
-                await scanner.prep()
+                await scanner._prep()
 
                 if not options.dry_run:
                     if not options.agent_mode and not options.yes and sys.stdin.isatty():
@@ -306,14 +308,25 @@ async def _main():
 
                     def keyboard_listen():
                         allowed_errors = 10
+                        kill_regex = re.compile(r"kill (?P<module>[a-z0-9_]+)")
                         while 1:
                             keyboard_input = "a"
                             try:
                                 keyboard_input = input()
                                 allowed_errors = 10
                             except Exception:
                                 allowed_errors -= 1
-                            if not keyboard_input:
+                            if keyboard_input:
+                                log.verbose(f'Got keyboard input: "{keyboard_input}"')
+                                kill_match = kill_regex.match(keyboard_input)
+                                if kill_match:
+                                    module = kill_match.group("module")
+                                    if module in scanner.modules:
+                                        log.hugewarning(f'Killing module: "{module}"')
+                                        scanner.manager.kill_module(module, message="killed by user")
+                                    else:
+                                        log.warning(f'Invalid module: "{module}"')
+                            else:
                                 toggle_log_level(logger=log)
                                 scanner.manager.modules_status(_log=True)
                             if allowed_errors <= 0:

bbot/core/__init__.py

@@ -1,4 +1,4 @@
 # logging
 from .logger import init_logging
 
-logging_queue, logging_handlers = init_logging()
+init_logging()

bbot/core/configurator/environ.py

@@ -107,8 +107,6 @@ def prepare_environment(bbot_config):
         # debug
         bbot_config["debug"] = args.cli_options.debug
         bbot_config["silent"] = args.cli_options.silent
-        if args.cli_options.output_dir:
-            bbot_config["output_dir"] = args.cli_options.output_dir
 
     import logging