Merge branch 'dev' into extract_internal_module

.github/workflows/distro_tests.yml

@@ -30,7 +30,7 @@ jobs:
             elif [ "$ID" = "arch" ]; then
               pacman -Syu --noconfirm curl git bash base-devel
             elif [ "$ID" = "fedora" ]; then
-              dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel
+              dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5
             elif [ "$ID" = "gentoo" ]; then
               echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
               emerge-webrsync

bbot/core/helpers/depsinstaller/installer.py

@@ -248,7 +248,7 @@ def tasks(self, module, tasks):
         return success
 
     def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
-        _ansible_args = {"ansible_connection": "local"}
+        _ansible_args = {"ansible_connection": "local", "ansible_python_interpreter": sys.executable}
         if ansible_args is not None:
             _ansible_args.update(ansible_args)
         module_args = None

bbot/modules/internal/speculate.py

@@ -1,6 +1,5 @@
 import random
 import ipaddress
-from pathlib import Path
 
 from bbot.core.helpers import validators
 from bbot.modules.internal.base import BaseInternalModule
@@ -24,21 +23,19 @@ class speculate(BaseInternalModule):
         "SOCIAL",
         "AZURE_TENANT",
         "USERNAME",
-        "FILESYSTEM",
     ]
-    produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB", "FILESYSTEM"]
+    produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"]
     flags = ["passive"]
     meta = {
         "description": "Derive certain event types from others by common sense",
         "created_date": "2022-05-03",
         "author": "@liquidsec",
     }
 
-    options = {"max_hosts": 65536, "ports": "80,443", "ignore_folders": [".git"]}
+    options = {"max_hosts": 65536, "ports": "80,443"}
     options_desc = {
         "max_hosts": "Max number of IP_RANGE hosts to convert into IP_ADDRESS events",
         "ports": "The set of ports to speculate on",
-        "ignore_folders": "Subfolders to ignore when crawling downloaded folders",
     }
     scope_distance_modifier = 1
     _priority = 4
@@ -75,13 +72,6 @@ async def setup(self):
                 self.hugewarning(f'Enabling the "portscan" module is highly recommended')
             self.range_to_ip = False
 
-        self.ignored_folders = self.config.get("ignore_folders", [])
-
-        return True
-
-    async def filter_event(self, event):
-        if event.type == "FILESYSTEM" and "folder" not in event.tags:
-            return False, "Event is not a folder"
         return True
 
     async def handle_event(self, event):
@@ -205,14 +195,3 @@ async def handle_event(self, event):
                 email_event = self.make_event(email, "EMAIL_ADDRESS", parent=event, tags=["affiliate"])
                 if email_event:
                     await self.emit_event(email_event, context="detected {event.type}: {event.data}")
-
-        # FILESYSTEM (folder) --> FILESYSTEM (files)
-        if event.type == "FILESYSTEM":
-            folder_path = Path(event.data["path"])
-            for file_path in folder_path.rglob("*"):
-                # If the file is not in an ignored folder and if it has an allowed extension raise it as a FILESYSTEM event
-                if not any(ignored_folder in str(file_path) for ignored_folder in self.ignored_folders):
-                    file_event = self.make_event(
-                        {"path": str(file_path)}, "FILESYSTEM", tags=["parsed_folder", "file"], parent=event
-                    )
-                    await self.emit_event(file_event)

bbot/modules/trufflehog.py

@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.83.1",
+        "version": "3.83.3",
         "config": "",
         "only_verified": True,
         "concurrency": 8,

bbot/scanner/preset/preset.py

@@ -376,6 +376,9 @@ def merge(self, other):
         # misc
         self.force_start = self.force_start | other.force_start
         self._cli = self._cli | other._cli
+        # transfer args
+        if other._args is not None:
+            self._args = other._args
 
     def bake(self, scan=None):
         """

bbot/test/test_step_2/module_tests/test_module_git_clone.py

@@ -215,7 +215,7 @@ class TestGit_CloneWithBlob(TestGit_Clone):
     config_overrides = {"folder_blobs": True}
 
     def check(self, module_test, events):
-        filesystem_events = [e for e in events if e.type == "FILESYSTEM" and "folder" in e.tags]
+        filesystem_events = [e for e in events if e.type == "FILESYSTEM"]
         assert len(filesystem_events) == 1
         assert all(["blob" in e.data for e in filesystem_events])
         filesystem_event = filesystem_events[0]

docs/data/chord_graph/entities.json

@@ -197,7 +197,6 @@
         "name": "FILESYSTEM",
         "parent": 88888888,
         "consumes": [
-            123,
             131,
             132
         ],
@@ -207,8 +206,7 @@
             72,
             78,
             81,
-            112,
-            123
+            112
         ]
     },
     {
@@ -1730,7 +1728,6 @@
             124,
             7,
             22,
-            10,
             2,
             12,
             110,
@@ -1742,7 +1739,6 @@
         ],
         "produces": [
             7,
-            10,
             4,
             12,
             15,

docs/data/chord_graph/rels.json

@@ -1344,11 +1344,6 @@
         "target": 22,
         "type": "consumes"
     },
-    {
-        "source": 123,
-        "target": 10,
-        "type": "consumes"
-    },
     {
         "source": 123,
         "target": 2,
@@ -1394,11 +1389,6 @@
         "target": 123,
         "type": "produces"
     },
-    {
-        "source": 10,
-        "target": 123,
-        "type": "produces"
-    },
     {
         "source": 4,
         "target": 123,