Merge pull request #1798 from blacklanternsecurity/dns-regex-yara-helper

YARA Helper to extract target hostnames from string

.github/workflows/tests.yml

@@ -58,7 +58,7 @@ jobs:
   update_docs:
     needs: test
     runs-on: ubuntu-latest
-    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
+    if: github.event_name == 'push' && (github.ref != 'refs/heads/dev' && github.ref != 'refs/heads/stable')
     steps:
       - uses: actions/checkout@v3
         with:

bbot/modules/internal/excavate.py

@@ -741,14 +741,8 @@ class HostnameExtractor(ExcavateRule):
 
         def __init__(self, excavate):
             super().__init__(excavate)
-            regexes_component_list = []
-            if excavate.scan.dns_regexes_yara:
-                for i, r in enumerate(excavate.scan.dns_regexes_yara):
-                    regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase")
-                regexes_component = " ".join(regexes_component_list)
-                self.yara_rules[f"hostname_extraction"] = (
-                    f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}'
-                )
+            if excavate.scan.dns_yara_rules_uncompiled:
+                self.yara_rules[f"hostname_extraction"] = excavate.scan.dns_yara_rules_uncompiled
 
         async def process(self, yara_results, event, yara_rule_settings, discovery_context):
             for identifier in yara_results.keys():

bbot/scanner/scanner.py

@@ -245,6 +245,8 @@ def __init__(
         self._dns_strings = None
         self._dns_regexes = None
         self._dns_regexes_yara = None
+        self._dns_yara_rules_uncompiled = None
+        self._dns_yara_rules = None
 
         self.__log_handlers = None
         self._log_handler_backup = []
@@ -1058,6 +1060,44 @@ def dns_regexes_yara(self):
             self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)+")
         return self._dns_regexes_yara
 
+    @property
+    def dns_yara_rules_uncompiled(self):
+        if self._dns_yara_rules_uncompiled is None:
+            regexes_component_list = []
+            for i, r in enumerate(self.dns_regexes_yara):
+                regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase")
+            if regexes_component_list:
+                regexes_component = " ".join(regexes_component_list)
+                self._dns_yara_rules_uncompiled = f'rule hostname_extraction {{meta: description = "matches DNS hostname pattern derived from target(s)" strings: {regexes_component} condition: any of them}}'
+        return self._dns_yara_rules_uncompiled
+
+    async def dns_yara_rules(self):
+        if self._dns_yara_rules is None:
+            if self.dns_yara_rules_uncompiled is not None:
+                import yara
+
+                self._dns_yara_rules = await self.helpers.run_in_executor(
+                    yara.compile, source=self.dns_yara_rules_uncompiled
+                )
+        return self._dns_yara_rules
+
+    async def extract_in_scope_hostnames(self, s):
+        """
+        Given a string, uses yara to extract hostnames matching scan targets
+
+        Examples:
+            >>> await self.scan.extract_in_scope_hostnames("http://www.evilcorp.com")
+            ... {"www.evilcorp.com"}
+        """
+        matches = set()
+        dns_yara_rules = await self.dns_yara_rules()
+        if dns_yara_rules is not None:
+            for match in await self.helpers.run_in_executor(dns_yara_rules.match, data=s):
+                for string in match.strings:
+                    for instance in string.instances:
+                        matches.add(str(instance))
+        return matches
+
     @property
     def json(self):
         """

bbot/test/test_step_1/test_regexes.py

@@ -372,3 +372,26 @@ async def test_regex_helper():
         assert matches.count(s) == 2
 
     await scan._cleanup()
+
+    # test yara hostname extractor helper
+    scan = Scanner("evilcorp.com", "www.evilcorp.net", "evilcorp.co.uk")
+    host_blob = """
+    https://asdf.evilcorp.com/
+    https://asdf.www.evilcorp.net/
+    https://asdf.www.evilcorp.co.uk/
+    https://asdf.www.evilcorp.com/
+    https://asdf.www.evilcorp.com/
+    """
+    extracted = await scan.extract_in_scope_hostnames(host_blob)
+    assert extracted == {
+        "asdf.www.evilcorp.net",
+        "asdf.evilcorp.com",
+        "asdf.www.evilcorp.com",
+        "www.evilcorp.com",
+        "asdf.www.evilcorp.co.uk",
+        "www.evilcorp.co.uk",
+    }
+
+    scan = Scanner()
+    extracted = await scan.extract_in_scope_hostnames(host_blob)
+    assert extracted == set()

docs/modules/nuclei.md

@@ -51,7 +51,7 @@ The Nuclei module has many configuration options:
 | modules.nuclei.silent         | bool   | Don't display nuclei's banner or status messages                                                                                                                                                                                                                                                                               | False     |
 | modules.nuclei.tags           | str    | execute a subset of templates that contain the provided tags                                                                                                                                                                                                                                                                   |           |
 | modules.nuclei.templates      | str    | template or template directory paths to include in the scan                                                                                                                                                                                                                                                                    |           |
-| modules.nuclei.version        | str    | nuclei version                                                                                                                                                                                                                                                                                                                 | 3.2.0     |
+| modules.nuclei.version        | str    | nuclei version                                                                                                                                                                                                                                                                                                                 | 3.3.2     |
 <!-- END BBOT MODULE OPTIONS NUCLEI -->
 
 Most of these you probably will **NOT** want to change. In particular, we advise against changing the version of Nuclei, as it's possible the latest version won't work right with BBOT.

docs/scanning/advanced.md

@@ -32,10 +32,12 @@ if __name__ == "__main__":
 
 <!-- BBOT HELP OUTPUT -->
 ```text
-usage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-p [PRESET ...]] [-c [CONFIG ...]] [-lp]
-               [-m MODULE [MODULE ...]] [-l] [-lmo] [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [--allow-deadly] [-n SCAN_NAME] [-v]
-               [-d] [-s] [--force] [-y] [--dry-run] [--current-preset] [--current-preset-full] [-o DIR] [-om MODULE [MODULE ...]] [--json] [--brief]
-               [--event-types EVENT_TYPES [EVENT_TYPES ...]] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [--version]
+usage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-b BLACKLIST [BLACKLIST ...]] [--strict-scope] [-p [PRESET ...]]
+               [-c [CONFIG ...]] [-lp] [-m MODULE [MODULE ...]] [-l] [-lmo] [-em MODULE [MODULE ...]] [-f FLAG [FLAG ...]] [-lf]
+               [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [--allow-deadly] [-n SCAN_NAME] [-v] [-d] [-s] [--force] [-y] [--dry-run]
+               [--current-preset] [--current-preset-full] [-o DIR] [-om MODULE [MODULE ...]] [--json] [--brief]
+               [--event-types EVENT_TYPES [EVENT_TYPES ...]]
+               [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [--version]
                [-H CUSTOM_HEADERS [CUSTOM_HEADERS ...]] [--custom-yara-rules CUSTOM_YARA_RULES]
 
 Bighuge BLS OSINT Tool
@@ -61,14 +63,14 @@ Presets:
 
 Modules:
   -m MODULE [MODULE ...], --modules MODULE [MODULE ...]
-                        Modules to enable. Choices: viewdns,postman,baddns_zone,dehashed,bucket_file_enum,asn,generic_ssrf,github_codesearch,columbus,azure_realm,dotnetnuke,dockerhub,credshed,passivetotal,certspotter,builtwith,otx,ipneighbor,fingerprintx,oauth,robots,dnsbrute_mutations,httpx,paramminer_headers,digitorus,gitlab,hunt,hunterio,trufflehog,ffuf,nuclei,badsecrets,git,bucket_firebase,ffuf_shortnames,urlscan,docker_pull,ip2location,subdomaincenter,telerik,pgp,zoomeye,shodan_dns,trickest,dnscommonsrv,ntlm,myssl,internetdb,emailformat,dastardly,azure_tenant,github_workflows,crt,affiliates,wayback,ajaxpro,wafw00f,iis_shortnames,sslcert,chaos,newsletters,host_header,bucket_amazon,vhost,paramminer_cookies,virustotal,rapiddns,leakix,dnsbrute,baddns,url_manipulation,code_repository,smuggler,bevigil,paramminer_getparams,unstructured,skymem,securitytrails,sitedossier,git_clone,bucket_azure,bucket_google,bypass403,wpscan,dnsdumpster,wappalyzer,dnscaa,social,hackertarget,github_org,fullhunt,filedownload,binaryedge,gowitness,anubisdb,portscan,ipstack,secretsdb,c99,censys,bucket_digitalocean
+                        Modules to enable. Choices: ntlm,robots,dockerhub,azure_tenant,crt,dnscommonsrv,dastardly,c99,hunt,skymem,dnscaa,gowitness,postman_download,dnsbrute,newsletters,secretsdb,nuclei,columbus,oauth,viewdns,shodan_dns,emailformat,gitlab,wappalyzer,internetdb,pgp,affiliates,bucket_file_enum,url_manipulation,ipneighbor,bucket_firebase,paramminer_cookies,virustotal,securitytxt,smuggler,dnsdumpster,dnsbrute_mutations,baddns,fingerprintx,paramminer_headers,wpscan,trufflehog,iis_shortnames,baddns_zone,dehashed,dotnetnuke,passivetotal,code_repository,generic_ssrf,portscan,censys,badsecrets,ipstack,bypass403,bucket_amazon,paramminer_getparams,github_workflows,github_codesearch,sslcert,otx,bucket_azure,fullhunt,postman,ffuf_shortnames,zoomeye,subdomaincenter,leakix,github_org,chaos,host_header,docker_pull,digitorus,unstructured,wafw00f,asn,credshed,vhost,trickest,binaryedge,bucket_google,filedownload,telerik,hunterio,httpx,ip2location,urlscan,git,hackertarget,git_clone,bevigil,wayback,certspotter,builtwith,ajaxpro,myssl,anubisdb,azure_realm,ffuf,rapiddns,securitytrails,bucket_digitalocean,sitedossier,social
   -l, --list-modules    List available modules.
   -lmo, --list-module-options
                         Show all module config options
   -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...]
                         Exclude these modules.
   -f FLAG [FLAG ...], --flags FLAG [FLAG ...]
-                        Enable modules by flag. Choices: subdomain-hijack,web-paramminer,subdomain-enum,code-enum,cloud-enum,iis-shortnames,web-thorough,baddns,portscan,slow,social-enum,affiliates,safe,web-screenshots,deadly,report,web-basic,email-enum,active,service-enum,aggressive,passive
+                        Enable modules by flag. Choices: slow,service-enum,baddns,subdomain-enum,deadly,web-thorough,iis-shortnames,report,affiliates,social-enum,email-enum,cloud-enum,web-basic,passive,web-screenshots,aggressive,web-paramminer,safe,subdomain-hijack,portscan,code-enum,active
   -lf, --list-flags     List available flags.
   -rf FLAG [FLAG ...], --require-flags FLAG [FLAG ...]
                         Only enable modules with these flags (e.g. -rf passive)
@@ -93,7 +95,7 @@ Output:
   -o DIR, --output-dir DIR
                         Directory to output scan results
   -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...]
-                        Output module(s). Choices: subdomains,emails,web_report,json,txt,websocket,slack,asset_inventory,neo4j,splunk,csv,stdout,http,python,discord,teams
+                        Output module(s). Choices: python,csv,subdomains,stdout,splunk,teams,emails,slack,http,websocket,discord,neo4j,web_report,json,asset_inventory,txt
   --json, -j            Output scan data in JSON format
   --brief, -br          Output only the data itself
   --event-types EVENT_TYPES [EVENT_TYPES ...]