fix wayback URL validation error

blacklanternsecurity · Sep 28, 2023 · 62d7a16 · 62d7a16
1 parent aaa3aba
commit 62d7a16
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/bbot/core/helpers/validators.py b/bbot/core/helpers/validators.py
@@ -143,7 +143,10 @@ def collapse_urls(urls, threshold=10):
     """
     url_hashes = {}
     for url in urls:
-        new_url = clean_url(url)
+        try:
+            new_url = clean_url(url)
+        except ValueError as e:
+            log.verbose(f"Failed to clean url {url}: {e}")
         url_hash = hash_url(new_url)
         try:
             url_hashes[url_hash].add(new_url)

diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
@@ -39,6 +39,8 @@ async def test_helpers_misc(helpers, scan, bbot_scanner, bbot_config, bbot_https
     assert helpers.validators.clean_url("http://evilcorp.com/asdf?a=asdf#frag").geturl() == "http://evilcorp.com/asdf"
     assert helpers.validators.clean_url("http://evilcorp.com//asdf").geturl() == "http://evilcorp.com/asdf"
     assert helpers.validators.clean_url("http://evilcorp.com.").geturl() == "http://evilcorp.com/"
+    with pytest.raises(ValueError):
+        helpers.validators.clean_url("http://evilcorp,com")
 
     assert helpers.url_depth("http://evilcorp.com/asdf/user/") == 2
     assert helpers.url_depth("http://evilcorp.com/asdf/user") == 2