fix conflicts

.github/workflows/tests.yml

@@ -15,19 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: psf/black@stable
-        with:
-          options: "--check"
-      - name: Install Python 3
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.x"
-      - name: Install dependencies
-        run: |
-          pip install flake8
-      - name: flake8
-        run: |
-          flake8
+      - run: |
+          pipx install ruff
+          ruff check
+          ruff format
   test:
     needs: lint
     runs-on: ubuntu-latest

.github/workflows/version_updater.yml

@@ -9,13 +9,13 @@ jobs:
   update-nuclei-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: dev
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -44,7 +44,7 @@ jobs:
         run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/deadly/nuclei.py"
       - name: Create pull request to update the version
         if: steps.update-version.outcome == 'success'
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           commit-message: "Update nuclei"
@@ -61,13 +61,13 @@ jobs:
   update-trufflehog-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: dev
           fetch-depth: 0
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
       - name: Install dependencies
@@ -96,7 +96,7 @@ jobs:
         run: "sed -i '0,/\"version\": \".*\",/ s/\"version\": \".*\",/\"version\": \"${{ env.latest_version }}\",/g' bbot/modules/trufflehog.py"
       - name: Create pull request to update the version
         if: steps.update-version.outcome == 'success'
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         with:
           token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
           commit-message: "Update trufflehog"
@@ -109,4 +109,4 @@ jobs:
           branch: "update-trufflehog"
           committer: blsaccess <[email protected]>
           author: blsaccess <[email protected]>
-          assignees: "TheTechromancer"
+          assignees: "TheTechromancer"

README.md

@@ -1,6 +1,6 @@
 [![bbot_banner](https://github.com/user-attachments/assets/f02804ce-9478-4f1e-ac4d-9cf5620a3214)](https://github.com/blacklanternsecurity/bbot)
 
-[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
+[![Python Version](https://img.shields.io/badge/python-3.9+-FF8400)](https://www.python.org) [![License](https://img.shields.io/badge/license-GPLv3-FF8400.svg)](https://github.com/blacklanternsecurity/bbot/blob/dev/LICENSE) [![DEF CON Recon Village 2024](https://img.shields.io/badge/DEF%20CON%20Demo%20Labs-2023-FF8400.svg)](https://www.reconvillage.org/talks) [![PyPi Downloads](https://static.pepy.tech/personalized-badge/bbot?right_color=orange&left_color=grey)](https://pepy.tech/project/bbot) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Tests](https://github.com/blacklanternsecurity/bbot/actions/workflows/tests.yml/badge.svg?branch=stable)](https://github.com/blacklanternsecurity/bbot/actions?query=workflow%3A"tests") [![Codecov](https://codecov.io/gh/blacklanternsecurity/bbot/branch/dev/graph/badge.svg?token=IR5AZBDM5K)](https://codecov.io/gh/blacklanternsecurity/bbot) [![Discord](https://img.shields.io/discord/859164869970362439)](https://discord.com/invite/PZqkgxu5SA)
 
 ### **BEE·bot** is a multipurpose scanner inspired by [Spiderfoot](https://github.com/smicallef/spiderfoot), built to automate your **Recon**, **Bug Bounties**, and **ASM**!
 

bbot/cli.py

@@ -29,7 +29,6 @@
 
 
 async def _main():
-
     import asyncio
     import traceback
     from contextlib import suppress
@@ -45,7 +44,6 @@ async def _main():
     global scan_name
 
     try:
-
         # start by creating a default scan preset
         preset = Preset(_log=True, name="bbot_cli_main")
         # parse command line arguments and merge into preset
@@ -81,7 +79,6 @@ async def _main():
 
         # if we're listing modules or their options
         if options.list_modules or options.list_module_options:
-
             # if no modules or flags are specified, enable everything
             if not (options.modules or options.output_modules or options.flags):
                 for module, preloaded in preset.module_loader.preloaded().items():
@@ -133,8 +130,8 @@ async def _main():
         ]
         if deadly_modules and not options.allow_deadly:
             log.hugewarning(f"You enabled the following deadly modules: {','.join(deadly_modules)}")
-            log.hugewarning(f"Deadly modules are highly intrusive")
-            log.hugewarning(f"Please specify --allow-deadly to continue")
+            log.hugewarning("Deadly modules are highly intrusive")
+            log.hugewarning("Please specify --allow-deadly to continue")
             return False
 
         # --current-preset
@@ -172,7 +169,6 @@ async def _main():
             log.trace(f"Command: {' '.join(sys.argv)}")
 
             if sys.stdin.isatty():
-
                 # warn if any targets belong directly to a cloud provider
                 for event in scan.target.seeds.events:
                     if event.type == "DNS_NAME":

bbot/core/config/files.py

@@ -10,7 +10,6 @@
 
 
 class BBOTConfigFiles:
-
     config_dir = (Path.home() / ".config" / "bbot").resolve()
     defaults_filename = (bbot_code_dir / "defaults.yml").resolve()
     config_filename = (config_dir / "bbot.yml").resolve()

bbot/core/config/logger.py

@@ -68,7 +68,7 @@ def __init__(self, core):
         self.listener = None
 
         # if we haven't set up logging yet, do it now
-        if not "_BBOT_LOGGING_SETUP" in os.environ:
+        if "_BBOT_LOGGING_SETUP" not in os.environ:
             os.environ["_BBOT_LOGGING_SETUP"] = "1"
             self.queue = multiprocessing.Queue()
             self.setup_queue_handler()

bbot/core/core.py

@@ -106,7 +106,7 @@ def default_config(self):
         if DEFAULT_CONFIG is None:
             self.default_config = self.files_config.get_default_config()
             # ensure bbot home dir
-            if not "home" in self.default_config:
+            if "home" not in self.default_config:
                 self.default_config["home"] = "~/.bbot"
         return DEFAULT_CONFIG
 

bbot/core/helpers/bloom.py

@@ -5,14 +5,14 @@
 
 class BloomFilter:
     """
-    Simple bloom filter implementation capable of rougly 400K lookups/s.
+    Simple bloom filter implementation capable of roughly 400K lookups/s.
 
     BBOT uses bloom filters in scenarios like DNS brute-forcing, where it's useful to keep track
     of which mutations have been tried so far.
 
     A 100-megabyte bloom filter (800M bits) can store 10M entries with a .01% false-positive rate.
     A python hash is 36 bytes. So if you wanted to store these in a set, this would take up
-    36 * 10M * 2 (key+value) == 720 megabytes. So we save rougly 7 times the space.
+    36 * 10M * 2 (key+value) == 720 megabytes. So we save roughly 7 times the space.
     """
 
     def __init__(self, size=8000000):

bbot/core/helpers/command.py

@@ -269,11 +269,11 @@ def _prepare_command_kwargs(self, command, kwargs):
         (['sudo', '-E', '-A', 'LD_LIBRARY_PATH=...', 'PATH=...', 'ls', '-l'], {'limit': 104857600, 'stdout': -1, 'stderr': -1, 'env': environ(...)})
     """
     # limit = 100MB (this is needed for cases like httpx that are sending large JSON blobs over stdout)
-    if not "limit" in kwargs:
+    if "limit" not in kwargs:
         kwargs["limit"] = 1024 * 1024 * 100
-    if not "stdout" in kwargs:
+    if "stdout" not in kwargs:
         kwargs["stdout"] = asyncio.subprocess.PIPE
-    if not "stderr" in kwargs:
+    if "stderr" not in kwargs:
         kwargs["stderr"] = asyncio.subprocess.PIPE
     sudo = kwargs.pop("sudo", False)
 
@@ -286,7 +286,7 @@ def _prepare_command_kwargs(self, command, kwargs):
 
     # use full path of binary, if not already specified
     binary = command[0]
-    if not "/" in binary:
+    if "/" not in binary:
         binary_full_path = which(binary)
         if binary_full_path is None:
             raise SubprocessError(f'Command "{binary}" was not found')

bbot/core/helpers/depsinstaller/installer.py

@@ -97,11 +97,11 @@ async def install(self, *modules):
                         or self.deps_behavior == "force_install"
                     ):
                         if not notified:
-                            log.hugeinfo(f"Installing module dependencies. Please be patient, this may take a while.")
+                            log.hugeinfo("Installing module dependencies. Please be patient, this may take a while.")
                             notified = True
                         log.verbose(f'Installing dependencies for module "{m}"')
                         # get sudo access if we need it
-                        if preloaded.get("sudo", False) == True:
+                        if preloaded.get("sudo", False) is True:
                             self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.')
                         success = await self.install_module(m)
                         self.setup_status[module_hash] = success
@@ -159,7 +159,7 @@ async def install_module(self, module):
         deps_common = preloaded["deps"]["common"]
         if deps_common:
             for dep_common in deps_common:
-                if self.setup_status.get(dep_common, False) == True:
+                if self.setup_status.get(dep_common, False) is True:
                     log.debug(
                         f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed'
                     )
@@ -244,7 +244,7 @@ def shell(self, module, commands):
         if success:
             log.info(f"Successfully ran {len(commands):,} shell commands")
         else:
-            log.warning(f"Failed to run shell dependencies")
+            log.warning("Failed to run shell dependencies")
         return success
 
     def tasks(self, module, tasks):
@@ -310,7 +310,7 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
         return success, err
 
     def read_setup_status(self):
-        setup_status = dict()
+        setup_status = {}
         if self.setup_status_cache.is_file():
             with open(self.setup_status_cache) as f:
                 with suppress(Exception):

bbot/core/helpers/diff.py

@@ -94,14 +94,14 @@ async def _baseline(self):
                 baseline_1_json = xmltodict.parse(baseline_1.text)
                 baseline_2_json = xmltodict.parse(baseline_2.text)
             except ExpatError:
-                log.debug(f"Cant HTML parse for {self.baseline_url}. Switching to text parsing as a backup")
+                log.debug(f"Can't HTML parse for {self.baseline_url}. Switching to text parsing as a backup")
                 baseline_1_json = baseline_1.text.split("\n")
                 baseline_2_json = baseline_2.text.split("\n")
 
             ddiff = DeepDiff(baseline_1_json, baseline_2_json, ignore_order=True, view="tree")
             self.ddiff_filters = []
 
-            for k, v in ddiff.items():
+            for k in ddiff.keys():
                 for x in list(ddiff[k]):
                     log.debug(f"Added {k} filter for path: {x.path()}")
                     self.ddiff_filters.append(x.path())
@@ -140,7 +140,7 @@ def compare_headers(self, headers_1, headers_2):
 
         ddiff = DeepDiff(headers_1, headers_2, ignore_order=True, view="tree")
 
-        for k, v in ddiff.items():
+        for k in ddiff.keys():
             for x in list(ddiff[k]):
                 try:
                     header_value = str(x).split("'")[1]
@@ -183,7 +183,7 @@ async def compare(
 
         await self._baseline()
 
-        if timeout == None:
+        if timeout is None:
             timeout = self.timeout
 
         reflection = False
@@ -203,7 +203,7 @@ async def compare(
         )
 
         if subject_response is None:
-            # this can be caused by a WAF not liking the header, so we really arent interested in it
+            # this can be caused by a WAF not liking the header, so we really aren't interested in it
             return (True, "403", reflection, subject_response)
 
         if check_reflection:
@@ -225,7 +225,7 @@ async def compare(
             subject_json = xmltodict.parse(subject_response.text)
 
         except ExpatError:
-            log.debug(f"Cant HTML parse for {subject.split('?')[0]}. Switching to text parsing as a backup")
+            log.debug(f"Can't HTML parse for {subject.split('?')[0]}. Switching to text parsing as a backup")
             subject_json = subject_response.text.split("\n")
 
         diff_reasons = []
@@ -238,11 +238,11 @@ async def compare(
 
         different_headers = self.compare_headers(self.baseline.headers, subject_response.headers)
         if different_headers:
-            log.debug(f"headers were different, no match")
+            log.debug("headers were different, no match")
             diff_reasons.append("header")
 
-        if self.compare_body(self.baseline_json, subject_json) == False:
-            log.debug(f"difference in HTML body, no match")
+        if self.compare_body(self.baseline_json, subject_json) is False:
+            log.debug("difference in HTML body, no match")
 
             diff_reasons.append("body")
 
@@ -275,6 +275,6 @@ async def canary_check(self, url, mode, rounds=3):
             )
 
             # if a nonsense header "caused" a difference, we need to abort. We also need to abort if our canary was reflected
-            if match == False or reflection == True:
+            if match is False or reflection is True:
                 return False
         return True

bbot/core/helpers/dns/brute.py

@@ -164,7 +164,7 @@ def gen_random_subdomains(self, n=50):
         for i in range(0, max(0, n - 5)):
             d = delimiters[i % len(delimiters)]
             l = lengths[i % len(lengths)]
-            segments = list(random.choice(self.devops_mutations) for _ in range(l))
+            segments = [random.choice(self.devops_mutations) for _ in range(l)]
             segments.append(self.parent_helper.rand_string(length=8, digits=False))
             subdomain = d.join(segments)
             yield subdomain

bbot/core/helpers/dns/dns.py

@@ -16,7 +16,6 @@
 
 
 class DNSHelper(EngineClient):
-
     SERVER_CLASS = DNSEngine
     ERROR_CLASS = DNSError
 
@@ -179,7 +178,7 @@ def _wildcard_prevalidation(self, host):
 
         host = clean_dns_record(host)
         # skip check if it's an IP or a plain hostname
-        if is_ip(host) or not "." in host:
+        if is_ip(host) or "." not in host:
             return False
 
         # skip if query isn't a dns name

bbot/core/helpers/dns/engine.py

@@ -24,7 +24,6 @@
 
 
 class DNSEngine(EngineServer):
-
     CMDS = {
         0: "resolve",
         1: "resolve_raw",
@@ -55,7 +54,7 @@ def __init__(self, socket_path, config={}, debug=False):
         dns_omit_queries = self.dns_config.get("omit_queries", None)
         if not dns_omit_queries:
             dns_omit_queries = []
-        self.dns_omit_queries = dict()
+        self.dns_omit_queries = {}
         for d in dns_omit_queries:
             d = d.split(":")
             if len(d) == 2:
@@ -73,7 +72,7 @@ def __init__(self, socket_path, config={}, debug=False):
             self.wildcard_ignore = []
         self.wildcard_ignore = tuple([str(d).strip().lower() for d in self.wildcard_ignore])
         self.wildcard_tests = self.dns_config.get("wildcard_tests", 5)
-        self._wildcard_cache = dict()
+        self._wildcard_cache = {}
         # since wildcard detection takes some time, This is to prevent multiple
         # modules from kicking off wildcard detection for the same domain at the same time
         self._wildcard_lock = NamedLock()
@@ -83,7 +82,7 @@ def __init__(self, socket_path, config={}, debug=False):
         self._last_connectivity_warning = time.time()
         # keeps track of warnings issued for wildcard detection to prevent duplicate warnings
         self._dns_warnings = set()
-        self._errors = dict()
+        self._errors = {}
         self._debug = self.dns_config.get("debug", False)
         self._dns_cache = LRUCache(maxsize=10000)
 
@@ -476,7 +475,6 @@ async def is_wildcard(self, query, rdtypes, raw_dns_records=None):
         # for every parent domain, starting with the shortest
         parents = list(domain_parents(query))
         for parent in parents[::-1]:
-
             # check if the parent domain is set up with wildcards
             wildcard_results = await self.is_wildcard_domain(parent, rdtypes_to_check)
 
@@ -640,7 +638,7 @@ async def _connectivity_check(self, interval=5):
                     self._last_dns_success = time.time()
                     return True
         if time.time() - self._last_connectivity_warning > interval:
-            self.log.warning(f"DNS queries are failing, please check your internet connection")
+            self.log.warning("DNS queries are failing, please check your internet connection")
             self._last_connectivity_warning = time.time()
         self._errors.clear()
         return False