Fix whitelisting scope bug

bbot/scanner/preset/preset.py

@@ -240,9 +240,8 @@ def __init__(
         from bbot.scanner.target import Target
 
         self.target = Target(*targets, strict_scope=self.strict_scope)
-        if not whitelist:
-            self.whitelist = self.target.copy()
-        else:
+        self.whitelist = None
+        if whitelist:
             self.whitelist = Target(*whitelist, strict_scope=self.strict_scope)
         if not blacklist:
             blacklist = []
@@ -326,11 +325,16 @@ def merge(self, other):
         self.flags.update(other.flags)
         # scope
         self.target.add(other.target)
-        self.whitelist.add(other.whitelist)
+        if other.whitelist:
+            if self.whitelist is None:
+                self.whitelist = other.whitelist.copy()
+            else:
+                self.whitelist.add(other.whitelist)
         self.blacklist.add(other.blacklist)
         self.strict_scope = self.strict_scope or other.strict_scope
         for t in (self.target, self.whitelist):
-            t.strict_scope = self.strict_scope
+            if t is not None:
+                t.strict_scope = self.strict_scope
         # log verbosity
         if other.silent:
             self.silent = other.silent
@@ -377,6 +381,10 @@ def bake(self):
         os.environ.clear()
         os.environ.update(os_environ)
 
+        # ensure whitelist
+        if baked_preset.whitelist is None:
+            baked_preset.whitelist = baked_preset.target.copy()
+
         # validate flags, config options
         baked_preset.validate()
 
@@ -615,8 +623,11 @@ def whitelisted(self, host):
             >>> preset.whitelisted("http://www.evilcorp.com")
             True
         """
+        whitelist = self.whitelist
+        if whitelist is None:
+            whitelist = self.target
         e = make_event(host, dummy=True)
-        return e in self.whitelist
+        return e in whitelist
 
     @classmethod
     def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):

bbot/test/test_step_1/test_cli.py

@@ -1,10 +1,90 @@
 from ..bbot_fixtures import *
 
+from bbot import cli
+
 
 @pytest.mark.asyncio
-async def test_cli_scan(monkeypatch):
-    from bbot import cli
+async def test_cli_scope(monkeypatch, capsys):
+    import json
+
+    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
+    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
+
+    # basic target without whitelist
+    monkeypatch.setattr(
+        "sys.argv",
+        ["bbot", "-t", "one.one.one.one", "-c", "scope_report_distance=10", "dns_resolution=true", "--json"],
+    )
+    result = await cli._main()
+    out, err = capsys.readouterr()
+    assert result == True
+    lines = [json.loads(l) for l in out.splitlines()]
+    dns_events = [l for l in lines if l["type"] == "DNS_NAME" and l["data"] == "one.one.one.one"]
+    assert dns_events
+    assert all([l["scope_distance"] == 0 and "in-scope" in l["tags"] for l in dns_events])
+    assert 1 == len(
+        [
+            l
+            for l in dns_events
+            if l["module"] == "TARGET"
+            and l["scope_distance"] == 0
+            and "in-scope" in l["tags"]
+            and "target" in l["tags"]
+        ]
+    )
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.1.1.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in ip_events])
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.0.0.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in ip_events])
+
+    # with whitelist
+    monkeypatch.setattr(
+        "sys.argv",
+        [
+            "bbot",
+            "-t",
+            "one.one.one.one",
+            "-w",
+            "192.168.0.1",
+            "-c",
+            "scope_report_distance=10",
+            "dns_resolution=true",
+            "scope_dns_search_distance=2",
+            "--json",
+        ],
+    )
+    result = await cli._main()
+    out, err = capsys.readouterr()
+    assert result == True
+    lines = [json.loads(l) for l in out.splitlines()]
+    lines = [l for l in lines if l["type"] != "SCAN"]
+    assert lines
+    assert not any([l["scope_distance"] == 0 for l in lines])
+    dns_events = [l for l in lines if l["type"] == "DNS_NAME" and l["data"] == "one.one.one.one"]
+    assert dns_events
+    assert all([l["scope_distance"] == 1 and "distance-1" in l["tags"] for l in dns_events])
+    assert 1 == len(
+        [
+            l
+            for l in dns_events
+            if l["module"] == "TARGET"
+            and l["scope_distance"] == 1
+            and "distance-1" in l["tags"]
+            and "target" in l["tags"]
+        ]
+    )
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.1.1.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 2 and "distance-2" in l["tags"] for l in ip_events])
+    ip_events = [l for l in lines if l["type"] == "IP_ADDRESS" and l["data"] == "1.0.0.1"]
+    assert ip_events
+    assert all([l["scope_distance"] == 2 and "distance-2" in l["tags"] for l in ip_events])
 
+
+@pytest.mark.asyncio
+async def test_cli_scan(monkeypatch):
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
 
@@ -50,8 +130,6 @@ async def test_cli_scan(monkeypatch):
 
 @pytest.mark.asyncio
 async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
-    from bbot import cli
-
     caplog.set_level(logging.INFO)
 
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
@@ -315,8 +393,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
 
 
 def test_cli_config_validation(monkeypatch, caplog):
-    from bbot import cli
-
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
 
@@ -338,8 +414,6 @@ def test_cli_config_validation(monkeypatch, caplog):
 
 
 def test_cli_module_validation(monkeypatch, caplog):
-    from bbot import cli
-
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
 
@@ -458,7 +532,6 @@ def test_cli_module_validation(monkeypatch, caplog):
 
 def test_cli_presets(monkeypatch, capsys, caplog):
     import yaml
-    from bbot import cli
 
     monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
     monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)

bbot/test/test_step_1/test_presets.py

@@ -212,6 +212,92 @@ def test_preset_scope():
     preset1.merge(preset4)
     set(preset1.output_modules) == {"python", "csv", "txt", "json", "stdout", "neo4j"}
 
+    # test preset merging + whitelist
+
+    preset_nowhitelist = Preset("evilcorp.com")
+    preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
+    assert preset_nowhitelist.in_scope("www.evilcorp.com")
+    assert not preset_nowhitelist.in_scope("www.evilcorp.de")
+    assert not preset_nowhitelist.in_scope("1.2.3.4/24")
+
+    assert "www.evilcorp.org" in preset_whitelist.target
+    assert "1.2.3.4" in preset_whitelist.whitelist
+    assert not preset_whitelist.in_scope("www.evilcorp.org")
+    assert not preset_whitelist.in_scope("www.evilcorp.de")
+    assert not preset_whitelist.whitelisted("www.evilcorp.org")
+    assert not preset_whitelist.whitelisted("www.evilcorp.de")
+    assert preset_whitelist.in_scope("1.2.3.4")
+    assert preset_whitelist.in_scope("1.2.3.4/28")
+    assert preset_whitelist.in_scope("1.2.3.4/24")
+    assert preset_whitelist.whitelisted("1.2.3.4")
+    assert preset_whitelist.whitelisted("1.2.3.4/28")
+    assert preset_whitelist.whitelisted("1.2.3.4/24")
+
+    assert set([e.data for e in preset_nowhitelist.target]) == {"evilcorp.com"}
+    assert preset_nowhitelist.whitelist is None
+    assert set([e.data for e in preset_whitelist.target]) == {"evilcorp.org"}
+    baked_nowhitelist = preset_nowhitelist.bake()
+    assert set([e.data for e in baked_nowhitelist.whitelist]) == {"evilcorp.com"}
+    baked_whitelist = preset_whitelist.bake()
+    assert set([e.data for e in baked_whitelist.whitelist]) == {"1.2.3.0/24"}
+
+    preset_nowhitelist.merge(preset_whitelist)
+    assert set([e.data for e in preset_nowhitelist.target]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_nowhitelist.whitelist]) == {"1.2.3.0/24"}
+    assert "www.evilcorp.org" in preset_nowhitelist.target
+    assert "www.evilcorp.com" in preset_nowhitelist.target
+    assert "1.2.3.4" in preset_nowhitelist.whitelist
+    assert not preset_nowhitelist.in_scope("www.evilcorp.org")
+    assert not preset_nowhitelist.in_scope("www.evilcorp.com")
+    assert not preset_nowhitelist.whitelisted("www.evilcorp.org")
+    assert not preset_nowhitelist.whitelisted("www.evilcorp.com")
+    assert preset_nowhitelist.in_scope("1.2.3.4")
+
+    preset_nowhitelist = Preset("evilcorp.com")
+    preset_whitelist = Preset("evilcorp.org", whitelist=["1.2.3.4/24"])
+    preset_whitelist.merge(preset_nowhitelist)
+    assert set([e.data for e in preset_whitelist.target]) == {"evilcorp.com", "evilcorp.org"}
+    assert set([e.data for e in preset_whitelist.whitelist]) == {"1.2.3.0/24"}
+    assert "www.evilcorp.org" in preset_whitelist.target
+    assert "www.evilcorp.com" in preset_whitelist.target
+    assert "1.2.3.4" in preset_whitelist.whitelist
+    assert not preset_whitelist.in_scope("www.evilcorp.org")
+    assert not preset_whitelist.in_scope("www.evilcorp.com")
+    assert not preset_whitelist.whitelisted("www.evilcorp.org")
+    assert not preset_whitelist.whitelisted("www.evilcorp.com")
+    assert preset_whitelist.in_scope("1.2.3.4")
+
+    preset_nowhitelist1 = Preset("evilcorp.com")
+    preset_nowhitelist2 = Preset("evilcorp.de")
+    assert set([e.data for e in preset_nowhitelist1.target]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2.target]) == {"evilcorp.de"}
+    assert preset_nowhitelist1.whitelist is None
+    assert preset_nowhitelist2.whitelist is None
+    preset_nowhitelist1.merge(preset_nowhitelist2)
+    assert set([e.data for e in preset_nowhitelist1.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in preset_nowhitelist2.target]) == {"evilcorp.de"}
+    assert preset_nowhitelist1.whitelist is None
+    assert preset_nowhitelist2.whitelist is None
+    assert "www.evilcorp.com" in preset_nowhitelist1.target
+    assert "www.evilcorp.de" in preset_nowhitelist1.target
+    assert preset_nowhitelist1.whitelisted("www.evilcorp.com")
+    assert preset_nowhitelist1.whitelisted("www.evilcorp.de")
+    assert not preset_nowhitelist1.whitelisted("1.2.3.4")
+    assert preset_nowhitelist1.in_scope("www.evilcorp.com")
+    assert preset_nowhitelist1.in_scope("www.evilcorp.de")
+    assert not preset_nowhitelist1.in_scope("1.2.3.4")
+
+    preset_nowhitelist1 = Preset("evilcorp.com")
+    preset_nowhitelist2 = Preset("evilcorp.de")
+    preset_nowhitelist2.merge(preset_nowhitelist1)
+    assert set([e.data for e in preset_nowhitelist1.target]) == {"evilcorp.com"}
+    assert set([e.data for e in preset_nowhitelist2.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert preset_nowhitelist1.whitelist is None
+    assert preset_nowhitelist2.whitelist is None
+    baked_nowhitelist2 = preset_nowhitelist2.bake()
+    assert set([e.data for e in baked_nowhitelist2.target]) == {"evilcorp.com", "evilcorp.de"}
+    assert set([e.data for e in baked_nowhitelist2.whitelist]) == {"evilcorp.com", "evilcorp.de"}
+
 
 def test_preset_logging():
     # test verbosity levels (conflicting verbose/debug/silent)