fixed tests

bbot/core/helpers/misc.py

@@ -106,11 +106,13 @@ def split_host_port(d):
     "192.168.1.1:443" --> (IPv4Address('192.168.1.1'), 443)
     "[dead::beef]:443" --> (IPv6Address('dead::beef'), 443)
     """
+    port = None
+    host = None
+    if is_ip(d):
+        return make_ip_type(d), port
     if not "://" in d:
         d = f"d://{d}"
     parsed = urlparse(d)
-    port = None
-    host = None
     with suppress(ValueError):
         if parsed.port is None:
             if parsed.scheme in ("https", "wss"):
@@ -938,12 +940,12 @@ def extract_host(s):
         after = s[match.end(1) :]
         host, port = split_host_port(hostname)
         if host is not None:
-            hostname = str(host)
             if port is not None:
                 after = f":{port}{after}"
-            if is_ip(hostname, version=6):
+            if is_ip(host, version=6) and hostname.startswith("["):
                 before = f"{before}["
                 after = f"]{after}"
+            hostname = str(host)
         return (hostname, before, after)
 
     return (None, s, "")

bbot/core/helpers/regexes.py

@@ -88,5 +88,5 @@
 jquery_post_regex = re.compile(r"\$.post\([\'\"].+[\'\"].+\{(.+)\}")
 a_tag_regex = re.compile(r"<a[^>]*href=[\"\'][^\"\'?>]*\?([^&\"\'=]+)")
 
-_extract_host_regex = r"(?:[a-z0-9]{1,20}://)?(?:[^?]*@)?([^\s!@#$%^&()=/?\\]+)"
+_extract_host_regex = r"(?:[a-z0-9]{1,20}://)?(?:[^?]*@)?([^\s!@#$%^&()=/?\\'\";~`<>]+)"
 extract_host_regex = re.compile(_extract_host_regex, re.I)

bbot/test/test_step_1/test_helpers.py

@@ -141,7 +141,7 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https
         "/my-file.csv",
     )
     assert helpers.extract_host("ftp://username:password:/@dead::beef/my-file.csv") == (
-        "my-ftp.com",
+        "dead::beef",
         "ftp://username:password:/@",
         "/my-file.csv",
     )
@@ -396,10 +396,6 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https
     assert helpers.smart_decode_punycode("[email protected]") == "bob_smith@ドメイン.テスト"
     assert helpers.smart_encode_punycode("ドメイン.テスト:80") == "xn--eckwd4c7c.xn--zckzah:80"
     assert helpers.smart_decode_punycode("xn--eckwd4c7c.xn--zckzah:80") == "ドメイン.テスト:80"
-    with pytest.raises(ValueError):
-        helpers.smart_decode_punycode(b"asdf")
-    with pytest.raises(ValueError):
-        helpers.smart_encode_punycode(b"asdf")
 
     assert helpers.recursive_decode("Hello%20world%21") == "Hello world!"
     assert helpers.recursive_decode("Hello%20%5Cu041f%5Cu0440%5Cu0438%5Cu0432%5Cu0435%5Cu0442") == "Hello Привет"