From 07a07138d382739ce8673cdb751f78b35fc1c292 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 26 Oct 2024 09:55:22 -0400 Subject: [PATCH 01/14] silence stderr in tests --- bbot/test/conftest.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 7cccc950d..516c0fec2 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -13,17 +13,31 @@ from bbot.core.helpers.misc import execute_sync_or_async from bbot.core.helpers.interactsh import server_list as interactsh_servers +# silence stdout + trace +root_logger = logging.getLogger() +pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log" +print(f"pytest_debug_file: {pytest_debug_file}") +debug_handler = logging.FileHandler(pytest_debug_file) +debug_handler.setLevel(logging.DEBUG) +debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s") +debug_handler.setFormatter(debug_format) +root_logger.addHandler(debug_handler) test_config = OmegaConf.load(Path(__file__).parent / "test.conf") -if test_config.get("debug", False): - os.environ["BBOT_DEBUG"] = "True" - logging.getLogger("bbot").setLevel(logging.DEBUG) - CORE.logger.log_level = logging.DEBUG -else: - # silence stdout + trace - root_logger = logging.getLogger() - for h in root_logger.handlers: - h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) + +os.environ["BBOT_DEBUG"] = "True" +CORE.logger.log_level = logging.DEBUG + +# silence all stderr output: +stderr_handler = CORE.logger.log_handlers["stderr"] +stderr_handler.setLevel(logging.CRITICAL) +handlers = list(CORE.logger.listener.handlers) +handlers.remove(stderr_handler) +CORE.logger.listener.handlers = tuple(handlers) + +for h in root_logger.handlers: + h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE")) + CORE.merge_default(test_config) From dd7884e7878f5c665689d2580b90604822bd7d41 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sat, 26 Oct 2024 09:58:27 -0400 Subject: [PATCH 02/14] upload debug logs --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a9aecd52..ab04a94e4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,6 +49,11 @@ jobs: - name: Run tests run: | poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot . + - name: Upload Debug Logs + uses: actions/upload-artifact@v3 + with: + name: pytest-debug-logs + path: pytest_debug.log - name: Upload Code Coverage uses: codecov/codecov-action@v3 with: From 7da0c63d91e6083b4414da825b2c7fbf0c2c559d Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 08:59:14 -0400 Subject: [PATCH 03/14] troubleshooting ubuntu 20.04 test --- .github/workflows/distro_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index e9eac869a..201f9a79a 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -60,4 +60,4 @@ jobs: export BBOT_DISTRO_TESTS=true poetry env use python3.11 poetry install - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG . + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO -k Testwpscan . From 9e90572446adf2742eeb55c43fb2ddfd54bea007 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 09:40:41 -0400 Subject: [PATCH 04/14] test things --- .github/workflows/distro_tests.yml | 2 +- bbot/core/helpers/depsinstaller/installer.py | 10 +++++++++- bbot/modules/wpscan.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 201f9a79a..128625654 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -33,7 +33,7 @@ jobs: dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel elif [ "$ID" = "gentoo" ]; then emerge-webrsync - emerge --update --newuse dev-vcs/git curl bash + emerge --update --newuse dev-vcs/git media-libs/mesa curl bash fi fi diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 8df5f05cf..80ded37a6 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -342,7 +342,15 @@ def install_core_deps(self): # ensure tldextract data is cached self.parent_helper.tldextract("evilcorp.co.uk") # command: package_name - core_deps = {"unzip": "unzip", "curl": "curl"} + core_deps = { + "unzip": "unzip", + "zipinfo": "unzip", + "curl": "curl", + "git": "git", + "make": "make", + "gcc": "gcc", + "bash": "bash", + } for command, package_name in core_deps.items(): if not self.parent_helper.which(command): to_install.add(package_name) diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index d9c43905e..4c1dce494 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -48,7 +48,7 @@ class wpscan(BaseModule): "name": "Install Ruby Deps (Fedora)", "package": {"name": ["rubygems", "ruby-devel"], "state": "present"}, "become": True, - "when": "ansible_facts['os_family'] == 'Fedora'", + "when": "ansible_facts['os_family'] == 'RedHat'", }, { "name": "Install wpscan gem", From 04411552da7760e3d7a73da04b48987b3b94049e Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 09:45:06 -0400 Subject: [PATCH 05/14] ruby things --- bbot/modules/wpscan.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index 4c1dce494..40c1d0181 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -33,10 +33,16 @@ class wpscan(BaseModule): deps_apt = ["curl", "make", "gcc"] deps_ansible = [ { - "name": "Install Ruby Deps (Debian/Ubuntu)", + "name": "Install Ruby Deps (Ubuntu 20.04)", + "package": {"name": ["ruby-dev"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian' and ansible_facts['distribution_version'] == '20.04'", + }, + { + "name": "Install Ruby Deps (Other Debian-based)", "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"}, "become": True, - "when": "ansible_facts['os_family'] == 'Debian'", + "when": "ansible_facts['os_family'] == 'Debian' and ansible_facts['distribution_version'] != '20.04'", }, { "name": "Install Ruby Deps (Arch)", From 5eded289ba0d2baa9d842cace96c2ad413ea65b3 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 09:58:59 -0400 Subject: [PATCH 06/14] more test things --- .github/workflows/distro_tests.yml | 1 + bbot/modules/wpscan.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 128625654..0d0515ac2 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -32,6 +32,7 @@ jobs: elif [ "$ID" = "fedora" ]; then dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel elif [ "$ID" = "gentoo" ]; then + echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd emerge-webrsync emerge --update --newuse dev-vcs/git media-libs/mesa curl bash fi diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index 40c1d0181..0dcd69fb3 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -36,13 +36,13 @@ class wpscan(BaseModule): "name": "Install Ruby Deps (Ubuntu 20.04)", "package": {"name": ["ruby-dev"], "state": "present"}, "become": True, - "when": "ansible_facts['os_family'] == 'Debian' and ansible_facts['distribution_version'] == '20.04'", + "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20'", }, { "name": "Install Ruby Deps (Other Debian-based)", "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"}, "become": True, - "when": "ansible_facts['os_family'] == 'Debian' and ansible_facts['distribution_version'] != '20.04'", + "when": "ansible_facts['distribution'] == 'Debian' and not (ansible_facts['os_family'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20')", }, { "name": "Install Ruby Deps (Arch)", From 95187e8abb0792a5e5edfaf73eb15fdf1a94667b Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 10:07:10 -0400 Subject: [PATCH 07/14] ansible things --- bbot/modules/wpscan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index 0dcd69fb3..c82c80b1d 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -42,7 +42,7 @@ class wpscan(BaseModule): "name": "Install Ruby Deps (Other Debian-based)", "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"}, "become": True, - "when": "ansible_facts['distribution'] == 'Debian' and not (ansible_facts['os_family'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20')", + "when": "ansible_facts['os_family'] == 'Debian' and not (ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20')", }, { "name": "Install Ruby Deps (Arch)", From 614dcd5d064c986b4e0ecde4a041b9da6384728d Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 10:16:25 -0400 Subject: [PATCH 08/14] ubuntu why --- bbot/modules/wpscan.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index c82c80b1d..295f10c0b 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -38,6 +38,12 @@ class wpscan(BaseModule): "become": True, "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20'", }, + { + "name": "Update RubyGems System (Ubuntu 20.04)", + "command": "gem update --system", + "become": True, + "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20'", + }, { "name": "Install Ruby Deps (Other Debian-based)", "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"}, From cfbd2116ab82fe6981db920aaa604a5ccb97c943 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 10:18:01 -0400 Subject: [PATCH 09/14] alpine --- bbot/modules/wpscan.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index 295f10c0b..c276c738f 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -62,6 +62,12 @@ class wpscan(BaseModule): "become": True, "when": "ansible_facts['os_family'] == 'RedHat'", }, + { + "name": "Install Ruby Deps (Alpine)", + "package": {"name": ["ruby-dev", "ruby-bundler"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Alpine'", + }, { "name": "Install wpscan gem", "gem": {"name": "wpscan", "state": "latest", "user_install": False}, From 991b531b556dfff03af90cea0a731e9a2d9d4a63 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 10:28:25 -0400 Subject: [PATCH 10/14] bye ubuntu --- .github/workflows/distro_tests.yml | 2 +- bbot/modules/wpscan.py | 16 ++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 0d0515ac2..957847a65 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"] + os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"] steps: - uses: actions/checkout@v4 - name: Install Python and Poetry diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index c276c738f..980d24fb5 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -33,22 +33,10 @@ class wpscan(BaseModule): deps_apt = ["curl", "make", "gcc"] deps_ansible = [ { - "name": "Install Ruby Deps (Ubuntu 20.04)", - "package": {"name": ["ruby-dev"], "state": "present"}, - "become": True, - "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20'", - }, - { - "name": "Update RubyGems System (Ubuntu 20.04)", - "command": "gem update --system", - "become": True, - "when": "ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20'", - }, - { - "name": "Install Ruby Deps (Other Debian-based)", + "name": "Install Ruby Deps (Debian)", "package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"}, "become": True, - "when": "ansible_facts['os_family'] == 'Debian' and not (ansible_facts['distribution'] == 'Ubuntu' and ansible_facts['distribution_major_version'] == '20')", + "when": "ansible_facts['os_family'] == 'Debian'", }, { "name": "Install Ruby Deps (Arch)", From f44983f6e1864b52f0c89e805fd31665ef7c1672 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 10:39:53 -0400 Subject: [PATCH 11/14] all tests --- .github/workflows/distro_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index 957847a65..e77e1453a 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -61,4 +61,4 @@ jobs: export BBOT_DISTRO_TESTS=true poetry env use python3.11 poetry install - poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO -k Testwpscan . + poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO . From 1f71c58c182ee77f159f473641246c5b6502bad0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 13:19:41 -0400 Subject: [PATCH 12/14] fix chrome in fedora --- bbot/core/config/logger.py | 1 - bbot/modules/gowitness.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 6a213d42d..5c48dd9f1 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -71,7 +71,6 @@ def __init__(self, core): # Start the QueueListener self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values()) self.listener.start() - atexit.register(self.listener.stop) self.log_level = logging.INFO diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 9d6d57483..571456302 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -72,7 +72,7 @@ async def setup(self): # make sure we have a working chrome install chrome_test_pass = False - for binary in ("chrome", "chromium", custom_chrome_path): + for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path): binary_path = self.helpers.which(binary) if binary_path and Path(binary_path).is_file(): chrome_test_proc = await self.run_process([binary_path, "--version"]) From f4bbd88c82a8f938e68bebbcfa539ed8365135b7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 13:23:11 -0400 Subject: [PATCH 13/14] goodbye gentoo, goodbye alpine --- .github/workflows/distro_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml index e77e1453a..395a38f5c 100644 --- a/.github/workflows/distro_tests.yml +++ b/.github/workflows/distro_tests.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"] + os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"] steps: - uses: actions/checkout@v4 - name: Install Python and Poetry From 8a8d33e01483d6128dbb3a3f5befbcdfe38b736c Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 27 Oct 2024 13:34:52 -0400 Subject: [PATCH 14/14] flaked --- bbot/core/config/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bbot/core/config/logger.py b/bbot/core/config/logger.py index 5c48dd9f1..4723d4237 100644 --- a/bbot/core/config/logger.py +++ b/bbot/core/config/logger.py @@ -1,5 +1,4 @@ import sys -import atexit import logging from copy import copy import multiprocessing