Merge pull request #1892 from blacklanternsecurity/better-logging

Log debug messages to file in tests

.github/workflows/distro_tests.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"]
+        os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"]
     steps:
       - uses: actions/checkout@v4
       - name: Install Python and Poetry
@@ -32,8 +32,9 @@ jobs:
             elif [ "$ID" = "fedora" ]; then
               dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel
             elif [ "$ID" = "gentoo" ]; then
+              echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
               emerge-webrsync
-              emerge --update --newuse dev-vcs/git curl bash
+              emerge --update --newuse dev-vcs/git media-libs/mesa curl bash
             fi
           fi
 
@@ -60,4 +61,4 @@ jobs:
           export BBOT_DISTRO_TESTS=true
           poetry env use python3.11
           poetry install
-          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG .
+          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .

.github/workflows/tests.yml

@@ -49,6 +49,11 @@ jobs:
       - name: Run tests
         run: |
           poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+      - name: Upload Debug Logs
+        uses: actions/upload-artifact@v3
+        with:
+          name: pytest-debug-logs
+          path: pytest_debug.log
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v3
         with:

bbot/core/config/logger.py

@@ -1,5 +1,4 @@
 import sys
-import atexit
 import logging
 from copy import copy
 import multiprocessing
@@ -71,7 +70,6 @@ def __init__(self, core):
             # Start the QueueListener
             self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values())
             self.listener.start()
-            atexit.register(self.listener.stop)
 
         self.log_level = logging.INFO
 

bbot/core/helpers/depsinstaller/installer.py

@@ -342,7 +342,15 @@ def install_core_deps(self):
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
         # command: package_name
-        core_deps = {"unzip": "unzip", "curl": "curl"}
+        core_deps = {
+            "unzip": "unzip",
+            "zipinfo": "unzip",
+            "curl": "curl",
+            "git": "git",
+            "make": "make",
+            "gcc": "gcc",
+            "bash": "bash",
+        }
         for command, package_name in core_deps.items():
             if not self.parent_helper.which(command):
                 to_install.add(package_name)

bbot/modules/gowitness.py

@@ -72,7 +72,7 @@ async def setup(self):
 
         # make sure we have a working chrome install
         chrome_test_pass = False
-        for binary in ("chrome", "chromium", custom_chrome_path):
+        for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path):
             binary_path = self.helpers.which(binary)
             if binary_path and Path(binary_path).is_file():
                 chrome_test_proc = await self.run_process([binary_path, "--version"])

bbot/modules/wpscan.py

@@ -33,7 +33,7 @@ class wpscan(BaseModule):
     deps_apt = ["curl", "make", "gcc"]
     deps_ansible = [
         {
-            "name": "Install Ruby Deps (Debian/Ubuntu)",
+            "name": "Install Ruby Deps (Debian)",
             "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"},
             "become": True,
             "when": "ansible_facts['os_family'] == 'Debian'",
@@ -48,7 +48,13 @@ class wpscan(BaseModule):
             "name": "Install Ruby Deps (Fedora)",
             "package": {"name": ["rubygems", "ruby-devel"], "state": "present"},
             "become": True,
-            "when": "ansible_facts['os_family'] == 'Fedora'",
+            "when": "ansible_facts['os_family'] == 'RedHat'",
+        },
+        {
+            "name": "Install Ruby Deps (Alpine)",
+            "package": {"name": ["ruby-dev", "ruby-bundler"], "state": "present"},
+            "become": True,
+            "when": "ansible_facts['os_family'] == 'Alpine'",
         },
         {
             "name": "Install wpscan gem",

bbot/test/conftest.py

@@ -13,17 +13,31 @@
 from bbot.core.helpers.misc import execute_sync_or_async
 from bbot.core.helpers.interactsh import server_list as interactsh_servers
 
+# silence stdout + trace
+root_logger = logging.getLogger()
+pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"
+print(f"pytest_debug_file: {pytest_debug_file}")
+debug_handler = logging.FileHandler(pytest_debug_file)
+debug_handler.setLevel(logging.DEBUG)
+debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
+debug_handler.setFormatter(debug_format)
+root_logger.addHandler(debug_handler)
 
 test_config = OmegaConf.load(Path(__file__).parent / "test.conf")
-if test_config.get("debug", False):
-    os.environ["BBOT_DEBUG"] = "True"
-    logging.getLogger("bbot").setLevel(logging.DEBUG)
-    CORE.logger.log_level = logging.DEBUG
-else:
-    # silence stdout + trace
-    root_logger = logging.getLogger()
-    for h in root_logger.handlers:
-        h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
+
+os.environ["BBOT_DEBUG"] = "True"
+CORE.logger.log_level = logging.DEBUG
+
+# silence all stderr output:
+stderr_handler = CORE.logger.log_handlers["stderr"]
+stderr_handler.setLevel(logging.CRITICAL)
+handlers = list(CORE.logger.listener.handlers)
+handlers.remove(stderr_handler)
+CORE.logger.listener.handlers = tuple(handlers)
+
+for h in root_logger.handlers:
+    h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))
+
 
 CORE.merge_default(test_config)