fix tests

blacklanternsecurity · Nov 6, 2024 · d0daea4 · d0daea4
1 parent a2a0123
commit d0daea4
Show file tree

Hide file tree

Showing 24 changed files with 27 additions and 35 deletions.
diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py
@@ -38,7 +38,7 @@ async def abort_if(self, event):
             return True, "DNS name is unresolved"
         return await super().abort_if(event)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json:

diff --git a/bbot/modules/binaryedge.py b/bbot/modules/binaryedge.py
@@ -37,6 +37,6 @@ async def request_url(self, query):
         url = f"{self.base_url}/query/domains/subdomain/{self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         return j.get("events", [])
diff --git a/bbot/modules/bufferoverrun.py b/bbot/modules/bufferoverrun.py
@@ -33,7 +33,7 @@ async def request_url(self, query):
         url = f"{self.commercial_base_url if self.commercial else self.base_url}?q=.{query}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):

diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py
@@ -26,7 +26,7 @@ async def request_url(self, query):
         url = f"{self.base_url}/subdomainfinder?key={{api_key}}&domain={self.helpers.quote(query)}&json"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         if isinstance(j, dict):
             subdomains = j.get("subdomains", [])

diff --git a/bbot/modules/certspotter.py b/bbot/modules/certspotter.py
@@ -17,7 +17,7 @@ def request_url(self, query):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
         return self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         json = r.json()
         if json:
             for r in json:

diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py
@@ -26,7 +26,7 @@ async def request_url(self, query):
         url = f"{self.base_url}/{domain}/subdomains"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         subdomains_set = set()
         if isinstance(j, dict):

diff --git a/bbot/modules/columbus.py b/bbot/modules/columbus.py
@@ -17,7 +17,7 @@ async def request_url(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}?days=365"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

diff --git a/bbot/modules/crt.py b/bbot/modules/crt.py
@@ -23,7 +23,7 @@ async def request_url(self, query):
         url = self.helpers.add_get_params(self.base_url, params).geturl()
         return await self.api_request(url, timeout=self.http_timeout + 30)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         for cert_info in j:
             if not type(cert_info) == dict:

diff --git a/bbot/modules/digitorus.py b/bbot/modules/digitorus.py
@@ -19,7 +19,7 @@ async def request_url(self, query):
         url = f"{self.base_url}/{self.helpers.quote(query)}"
         return await self.helpers.request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         content = getattr(r, "text", "")
         extract_regex = re.compile(r"[\w.-]+\." + query, re.I)

diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py
@@ -35,5 +35,5 @@ async def request_url(self, query):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         return r.json().get("hosts", [])
diff --git a/bbot/modules/hackertarget.py b/bbot/modules/hackertarget.py
@@ -18,7 +18,7 @@ async def request_url(self, query):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         for line in r.text.splitlines():
             host = line.split(",")[0]
             try:

diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py
@@ -36,7 +36,7 @@ async def request_url(self, query):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
         json = r.json()
         if json:
             for entry in json:

diff --git a/bbot/modules/myssl.py b/bbot/modules/myssl.py
@@ -17,7 +17,7 @@ async def request_url(self, query):
         url = f"{self.base_url}?domain={self.helpers.quote(query)}"
         return await self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, dict):

diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py
@@ -17,7 +17,7 @@ def request_url(self, query):
         url = f"{self.base_url}/api/v1/indicators/domain/{self.helpers.quote(query)}/passive_dns"
         return self.api_request(url)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         if isinstance(j, dict):
             for entry in j.get("passive_dns", []):

diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py
@@ -37,7 +37,7 @@ async def request_url(self, query):
         url = f"{self.base_url}/enrichment/subdomains?query={self.helpers.quote(query)}"
         return await self.api_request(url, auth=self.auth)
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         for subdomain in r.json().get("subdomains", []):
             yield f"{subdomain}.{query}"
 

diff --git a/bbot/modules/rapiddns.py b/bbot/modules/rapiddns.py
@@ -18,11 +18,7 @@ async def request_url(self, query):
         response = await self.api_request(url, timeout=self.http_timeout + 10)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)
diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py
@@ -26,7 +26,7 @@ async def request_url(self, query):
         response = await self.api_request(url)
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         j = r.json()
         if isinstance(j, dict):
             for host in j.get("subdomains", []):

diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py
@@ -22,5 +22,5 @@ async def handle_event(self, event):
     def make_url(self, query):
         return f"{self.base_url}/dns/domain/{self.helpers.quote(query)}?key={{api_key}}&page={{page}}"
 
-    def parse_results(self, json, query):
+    async def parse_results(self, json, query):
         return [f"{sub}.{query}" for sub in json.get("subdomains", [])]
diff --git a/bbot/modules/subdomaincenter.py b/bbot/modules/subdomaincenter.py
@@ -33,7 +33,7 @@ async def request_url(self, query):
                 break
         return response
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         json = r.json()
         if json and isinstance(json, list):

diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py
@@ -106,7 +106,7 @@ def make_query(self, event):
                 break
         return ".".join([s for s in query.split(".") if s != "_wildcard"])
 
-    def parse_results(self, r, query=None):
+    async def parse_results(self, r, query=None):
         json = r.json()
         if json:
             for hostname in json:
@@ -123,7 +123,7 @@ async def query(self, query, request_fn=None, parse_fn=None):
                 self.info(f'Query "{query}" failed (no response)')
                 return []
             try:
-                results = list(parse_fn(response, query))
+                results = list(await parse_fn(response, query))
             except Exception as e:
                 if response:
                     self.info(

diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py
@@ -36,7 +36,7 @@ def make_url(self, query):
         url += "&limit={page_size}&offset={offset}&select=hostname&orderby=hostname"
         return url
 
-    def parse_results(self, j, query):
+    async def parse_results(self, j, query):
         results = j.get("results", [])
         subdomains = set()
         for item in results:

diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py
@@ -24,11 +24,7 @@ def prepare_api_request(self, url, kwargs):
         kwargs["headers"]["x-apikey"] = self.api_key
         return url, kwargs
 
-    def parse_results(self, r, query):
+    async def parse_results(self, r, query):
         results = set()
         text = getattr(r, "text", "")
-        for match in self.helpers.regexes.dns_name_regex.findall(text):
-            match = match.lower()
-            if match.endswith(query):
-                results.add(match)
-        return results
+        return await self.scan.extract_in_scope_hostnames(text)
diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py
@@ -70,6 +70,6 @@ async def query(self, query):
             agen.aclose()
         return results
 
-    def parse_results(self, r):
+    async def parse_results(self, r):
         for entry in r.get("list", []):
             yield entry["name"]
diff --git a/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py b/bbot/test/test_step_2/module_tests/test_module_ffuf_shortnames.py
@@ -142,7 +142,7 @@ async def setup_after_prep(self, module_test):
                 tags=["shortname-file"],
             )
         )
-        module_test.scan.target.seeds._events = set(seed_events)
+        module_test.scan.target.seeds.events = set(seed_events)
 
         expect_args = {"method": "GET", "uri": "/administrator.aspx"}
         respond_args = {"response_data": "alive"}