Merge pull request #1885 from blacklanternsecurity/dev

Dev -> Stable 2.1.1

.flake8

@@ -2,4 +2,4 @@
 select = F,E722
 ignore = F403,F405,F541
 per-file-ignores =
-    */__init__.py:F401,F403
+    */__init__.py:F401,F403

.github/workflows/distro_tests.yml

@@ -0,0 +1,64 @@
+name: Tests (Linux Distros)
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-distros:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Python and Poetry
+        run: |
+          if [ -f /etc/os-release ]; then
+            . /etc/os-release
+            if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update
+              apt-get -y install curl git bash build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
+            elif [ "$ID" = "alpine" ]; then
+              apk add --no-cache bash gcc g++ musl-dev libffi-dev curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
+            elif [ "$ID" = "arch" ]; then
+              pacman -Syu --noconfirm curl git bash base-devel
+            elif [ "$ID" = "fedora" ]; then
+              dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel
+            elif [ "$ID" = "gentoo" ]; then
+              echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
+              emerge-webrsync
+              emerge --update --newuse dev-vcs/git media-libs/mesa curl bash
+            fi
+          fi
+
+          # Re-run the script with bash
+          exec bash -c "
+            curl https://pyenv.run | bash
+            export PATH=\"$HOME/.pyenv/bin:\$PATH\"
+            export PATH=\"$HOME/.local/bin:\$PATH\"
+            eval \"\$(pyenv init --path)\"
+            eval \"\$(pyenv init -)\"
+            eval \"\$(pyenv virtualenv-init -)\"
+            pyenv install 3.11
+            pyenv global 3.11
+            pyenv rehash
+            python3.11 -m pip install --user pipx
+            python3.11 -m pipx ensurepath
+            pipx install poetry
+          "
+      - name: Run tests
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          export PATH="$HOME/.pyenv/bin:$PATH"
+          export PATH="$HOME/.pyenv/shims:$PATH"
+          export BBOT_DISTRO_TESTS=true
+          poetry env use python3.11
+          poetry install
+          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .

.github/workflows/docs_updater.yml

@@ -2,7 +2,8 @@ name: Daily Docs Update
 
 on:
   schedule:
-    - cron: '0 0 * * *'  # Runs daily at midnight UTC
+    - cron: '30 2 * * *'  # Runs daily at 2:30 AM UTC, a less congested time
+  workflow_dispatch:      # Allows manual triggering of the workflow
 
 jobs:
   update_docs:
@@ -23,18 +24,22 @@ jobs:
       - name: Generate docs
         run: |
           poetry run bbot/scripts/docs.py
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v9
-        with:
-          add: '["*.md", "docs/data/chord_graph/*.json"]'
-          author_name: "BBOT Docs Autopublish"
-          author_email: [email protected]
-          message: "Refresh module docs"
-      - name: Create Pull Request
+      - name: Commit and Push Changes
+        run: |
+          git config user.name "BBOT Docs Autopublish"
+          git config user.email "[email protected]"
+          git checkout -b update-docs
+          git add "*.md" "docs/data/chord_graph/*.json"
+          git commit -m "Refresh module docs"
+          git push -u origin update-docs --force
+        env:
+          GITHUB_TOKEN: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+      - name: Create or Update Pull Request
         uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           branch: update-docs
           base: dev
           title: "Daily Docs Update"
           body: "This is an automated pull request to update the documentation."
+          update-existing: true

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: tests
+name: Tests (Python Versions)
 on:
   push:
     branches:
@@ -14,20 +14,20 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check"
       - name: Install Python 3
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
         run: |
           pip install flake8
       - name: flake8
         run: |
-          flake8 --select F,E722 --ignore F403,F405,F541 --per-file-ignores="*/__init__.py:F401,F403"
+          flake8
   test:
     needs: lint
     runs-on: ubuntu-latest
@@ -37,9 +37,9 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -48,95 +48,29 @@ jobs:
           poetry install
       - name: Run tests
         run: |
-          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+      - name: Upload Debug Logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: pytest-debug-logs
+          path: pytest_debug.log
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
           verbose: true
-  update_docs:
-    needs: test
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable')
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install
-      - name: Generate docs
-        run: |
-          poetry run bbot/scripts/docs.py
-      - name: Commit docs
-        uses: EndBug/add-and-commit@v9
-        continue-on-error: true
-        with:
-          add: '["*.md", "docs/data/chord_graph/*.json"]'
-          author_name: "BBOT Docs Autopublish"
-          author_email: [email protected]
-          message: "Refresh module docs"
-  publish_docs:
-    needs: test
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev')
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
-      - uses: actions/cache@v3
-        with:
-          key: mkdocs-material-${{ env.cache_id }}
-          path: .cache
-          restore-keys: |
-            mkdocs-material-
-      - name: Install dependencies
-        run: |
-          pip install poetry
-          poetry install --only=docs
-      - name: Configure Git
-        run: |
-          git config user.name github-actions
-          git config user.email [email protected]
-          git fetch origin gh-pages:refs/remotes/origin/gh-pages
-          if git show-ref --verify --quiet refs/heads/gh-pages; then
-            git branch -f gh-pages origin/gh-pages
-          else
-            git branch --track gh-pages origin/gh-pages
-          fi
-      - name: Generate docs (stable branch)
-        if: github.ref == 'refs/heads/stable'
-        run: |
-          poetry run mike deploy Stable
-      - name: Generate docs (dev branch)
-        if: github.ref == 'refs/heads/dev'
-        run: |
-          poetry run mike deploy Dev
-      - name: Publish docs
-        run: |
-          git switch gh-pages
-          git push
   publish_code:
     needs: test
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -185,7 +119,7 @@ jobs:
   #   runs-on: ubuntu-latest
   #   if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
   #   steps:
-  #     - uses: actions/checkout@v3
+  #     - uses: actions/checkout@v4
   #       with:
   #         ref: ${{ github.head_ref }}
   #         fetch-depth: 0 # Fetch all history for all tags and branches

bbot/core/config/logger.py

@@ -5,6 +5,7 @@
 import multiprocessing
 import logging.handlers
 from pathlib import Path
+from contextlib import suppress
 
 from ..helpers.misc import mkdir, error_and_exit
 from ...logger import colorize, loglevel_mapping
@@ -71,10 +72,36 @@ def __init__(self, core):
             # Start the QueueListener
             self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values())
             self.listener.start()
-            atexit.register(self.listener.stop)
+            atexit.register(self.cleanup_logging)
 
         self.log_level = logging.INFO
 
+    def cleanup_logging(self):
+        # Close the queue handler
+        with suppress(Exception):
+            self.queue_handler.close()
+
+        # Clean root logger
+        root_logger = logging.getLogger()
+        for handler in list(root_logger.handlers):
+            with suppress(Exception):
+                root_logger.removeHandler(handler)
+            with suppress(Exception):
+                handler.close()
+
+        # Clean all other loggers
+        for logger in logging.Logger.manager.loggerDict.values():
+            if hasattr(logger, "handlers"):  # Logger, not PlaceHolder
+                for handler in list(logger.handlers):
+                    with suppress(Exception):
+                        logger.removeHandler(handler)
+                    with suppress(Exception):
+                        handler.close()
+
+        # Stop queue listener
+        with suppress(Exception):
+            self.listener.stop()
+
     def setup_queue_handler(self, logging_queue=None, log_level=logging.DEBUG):
         if logging_queue is None:
             logging_queue = self.queue

bbot/core/helpers/depsinstaller/installer.py

@@ -342,7 +342,15 @@ def install_core_deps(self):
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
         # command: package_name
-        core_deps = {"unzip": "unzip", "curl": "curl"}
+        core_deps = {
+            "unzip": "unzip",
+            "zipinfo": "unzip",
+            "curl": "curl",
+            "git": "git",
+            "make": "make",
+            "gcc": "gcc",
+            "bash": "bash",
+        }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):
                 to_install.add(package_name)

bbot/modules/bufferoverrun.py

@@ -0,0 +1,48 @@
+from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey
+
+
+class BufferOverrun(subdomain_enum_apikey):
+    watched_events = ["DNS_NAME"]
+    produced_events = ["DNS_NAME"]
+    flags = ["subdomain-enum", "passive", "safe"]
+    meta = {
+        "description": "Query BufferOverrun's TLS API for subdomains",
+        "created_date": "2024-10-23",
+        "author": "@TheTechromancer",
+        "auth_required": True,
+    }
+    options = {"api_key": "", "commercial": False}
+    options_desc = {"api_key": "BufferOverrun API key", "commercial": "Use commercial API"}
+
+    base_url = "https://tls.bufferover.run/dns"
+    commercial_base_url = "https://bufferover-run-tls.p.rapidapi.com/ipv4/dns"
+
+    async def setup(self):
+        self.commercial = self.config.get("commercial", False)
+        return await super().setup()
+
+    def prepare_api_request(self, url, kwargs):
+        if self.commercial:
+            kwargs["headers"]["x-rapidapi-host"] = "bufferover-run-tls.p.rapidapi.com"
+            kwargs["headers"]["x-rapidapi-key"] = self.api_key
+        else:
+            kwargs["headers"]["x-api-key"] = self.api_key
+        return url, kwargs
+
+    async def request_url(self, query):
+        url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
+        return await self.api_request(url)
+
+    def parse_results(self, r, query):
+        j = r.json()
+        subdomains_set = set()
+        if isinstance(j, dict):
+            results = j.get("Results", [])
+            for result in results:
+                parts = result.split(",")
+                if len(parts) > 4:
+                    subdomain = parts[4].strip()
+                    if subdomain and subdomain.endswith(f".{query}"):
+                        subdomains_set.add(subdomain)
+            for subdomain in subdomains_set:
+                yield subdomain

bbot/modules/deadly/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.3.4",
+        "version": "3.3.5",
         "tags": "",
         "templates": "",
         "severity": "",