diff --git a/bbot/presets/spider.yml b/bbot/presets/spider.yml index 0ffb495c48..14561ed353 100644 --- a/bbot/presets/spider.yml +++ b/bbot/presets/spider.yml @@ -3,6 +3,10 @@ description: Recursive web spider modules: - httpx +blacklist: + # Prevent spider from invalidating sessions by logging out + - "RE:/.*(sign[_-]?out|log[_-]?out)" + config: web: # how many links to follow in a row diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 8442e77107..cd13209b29 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -266,7 +266,7 @@ async def _prep(self): f.write(self.preset.to_yaml()) # log scan overview - start_msg = f"Scan with {len(self.preset.scan_modules):,} modules seeded with {len(self.seeds):,} targets" + start_msg = f"Scan seeded with {len(self.seeds):,} targets" details = [] if self.whitelist != self.target: details.append(f"{len(self.whitelist):,} in whitelist") diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 68067cee0a..09541d183b 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -188,6 +188,7 @@ def __init__(self, *args, **kwargs): @special_target_type(r"^(?:RE|REGEX):(.*)") def handle_regex(self, match): pattern = match.group(1) + log.info(f"Blacklisting by custom regex: {pattern}") blacklist_regex = re.compile(pattern, re.IGNORECASE) self.blacklist_regexes.add(blacklist_regex) return []