Merge pull request #2 from leunammejii/add_tor

Add support for Torsocks

README.md

@@ -9,7 +9,9 @@ After catching malicious phishing domain names using [certstream](https://certst
 Search for specific filetypes submitted to [urlscan.io](https://urlscan.io/) and recursively download the webpage if predefined file extensions are found.  
 
 #### Prerequisites  
+- Ubuntu 18.04+ (should work on other Linux distros)  
 - Python 2.7.14  
+- Torsocks (optional: used with flag `--tor`)  
 
 #### Setup  
 1. Open a terminal and run the following command:  
@@ -27,8 +29,12 @@ The following command will:
 - Score and add suspicious domains to a queue while other domains continue to be scored  
 - Simultaneously make requests to the domains in the queue to search for predefined file extensions  
 - Recursively download the site when an open directory is found hosting a file with a particular extension  
+
+Optional arguments:  
+- **--timeout** : Set time to wait for a connection  
+- **--tor** : Download files via the Tor network  
 ```bash
-python opendir_certstream.py
+python opendir_certstream.py [--timeout] [--tor]
 ```
 **Note**: Any URLs in the queue will be lost once the program stops.  
 
@@ -43,10 +49,12 @@ The following command will:
 - **File Extension** : 7z, apk, bat, bz, bz2, crypt, dll, doc, docx, exe, gz, hta, iso, jar, json, lnk, ppt, ps1, py, rar, sfx, sh, tar, vb, vbs, xld, xls, xlsx, zip   
 
 Optional arguments:  
-- **Dry Run** : Perform a test run to see what would be downloaded  
-- **Exclude** : A comma-separated list of domains to not download content from (ex. 'google.com,bing.com')  
+- **--dryrun** : Perform a test run to see what would be downloaded  
+- **--exclude** : A comma-separated list of domains to not download content from (ex. 'google.com,bing.com')  
+- **--timeout** : Set time to wait for a connection  
+- **--tor** : Download files via the Tor network  
 ```bash
-python opendir_urlscan.py <QUERY_TYPE> <DELTA> <FILE_EXTENSION> [--dry-run] [--exclude=CSV]
+python opendir_urlscan.py <QUERY_TYPE> <DELTA> <FILE_EXTENSION> [--dry-run] [--exclude=CSV] [--timeout] [--tor]
 ```
 **Note**: If the path is a file, it will be downloaded regardless of whether it's an open directory.  
 
@@ -60,6 +68,8 @@ python opendir_urlscan.py <QUERY_TYPE> <DELTA> <FILE_EXTENSION> [--dry-run] [--e
 ![opendir_urlscan - Download](https://github.com/leunammejii/analyst_arsenal/blob/master/static/assets/opendir_urlscan_download.png)  
 
 #### Things to know  
-- Be responsible.  
+- Be responsible!!!  
+- Downloads via Tor happen over **127.0.0.1:9050**  
+- These scripts **will not** check Torsocks settings  
 
 Please fork, create merge requests, and help make this better.  

external.yaml

@@ -46,3 +46,12 @@ tlds:
 # Add your own TLDs here, e.g.:
 #    '.nu':
 #    '.se':
+
+queries:
+    'automatic' : 'task.method%3Aautomatic'
+    'manual' : 'task.method%3Amanual'
+    'certstream' : '(task.source%3Acertstream-idn OR task.source%3Acertstream-suspicious)'
+    'openphish' : 'task.source%3Aopenphish'
+    'phishtank' : 'task.source%3Aphishtank'
+    'twitter' : '(task.source%3Atwitter OR task.source%3Atwitter_illegalFawn OR task.source%3Atwitter_phishingalert)'
+    'urlhaus' : 'task.source%3Aurlhaus'

opendir_certstream.py

@@ -6,8 +6,17 @@
 - Simultaneously make requests to the domains in the queue to search for predefined file extensions
 - Recursively download the site when an open directory is found hosting a file with a particular extension
 
+Optional arguments:
+- --timeout : Set time to wait for a connection
+- --tor     : Download files via the Tor network
+
 Credit: https://github.com/x0rz/phishing_catcher
 
+Resources:
+    http://docs.python-requests.org/en/master/user/advanced/#proxies
+    https://gist.github.com/jefftriplett/9748036
+    https://ec.haxx.se/libcurl-proxies.html
+
 Usage:
 
 ```
@@ -17,6 +26,7 @@
 Debugger: open("/tmp/splunk_script.txt", "a").write("{}: <MSG>\n".format(<VAR>))
 """
 
+import argparse
 import os
 import Queue
 import re
@@ -39,12 +49,20 @@
 from confusables import unconfuse
 
 
-certstream_url = "wss://certstream.calidog.io"
-pbar           = tqdm.tqdm(desc="certificate_update", unit="cert")
-uagent         = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
-
-global url_queue
-url_queue = Queue.Queue()
+# Parse Arguments
+parser = argparse.ArgumentParser(description="Attempt to detect phishing kits and open directories via Certstream.")
+parser.add_argument("--timeout",
+                    dest="timeout",
+                    type=int,
+                    default=30,
+                    required=False,
+                    help="Set time to wait for a connection")
+parser.add_argument("--tor",
+                    dest="tor",
+                    action="store_true",
+                    required=False,
+                    help="Download files over the Tor network")
+args = parser.parse_args()
 
 # hxxp://sebastiandahlgren[.]se/2014/06/27/running-a-method-as-a-background-thread-in-python/
 class QueueManager(object):
@@ -75,10 +93,15 @@ def run(self):
                 url   = url_queue.get()
                 tqdm.tqdm.write(
                     "[*] Session   : "
-                    "{}".format(colored(url, "blue")))
+                    "{}".format(colored(url, "blue"))
+                )
                 try:
-                    resp = requests.get(url, headers={"User-Agent": uagent}, timeout=3.1)
-                except:
+                    resp = requests.get(url,
+                                        proxies=proxies,
+                                        headers={"User-Agent": uagent},
+                                        timeout=timeout,
+                                        allow_redirects=True)
+                except Exception as err:
                     continue
 
                 if not (resp.status_code == 200 and "Index of " in resp.content):
@@ -96,15 +119,18 @@ def run(self):
 
                     tqdm.tqdm.write(
                         "[*] Download  : "
-                        "{}".format(colored(url, "green", attrs=["underline", "bold"])))
+                        "{} ('Index of ' found)".format(
+                            colored(url, "green", attrs=["underline", "bold"]))
+                    )
 
                     try:
                         subprocess.call([
+                            "{}".format(torsocks),
                             "wget",
                             "--execute=robots=off",
                             "--tries=2",
                             "--no-clobber",
-                            "--timeout=3.1",
+                            "--timeout={}".format(timeout),
                             "--waitretry=0",
                             "--directory-prefix=./{}/".format(directory),
                             "--content-disposition",
@@ -113,9 +139,11 @@ def run(self):
                             "--no-parent",
                             url
                         ])
-                        exit(0)
                         break
-                    except:
+                    except Exception as err:
+                        print("[!] Error    : {}".format(
+                            colored(err, "red", attrs=["bold"])
+                        ))
                         continue
             time.sleep(self.interval)        
 
@@ -178,8 +206,13 @@ def score_domain(domain):
 
     try:
         res = get_tld(domain, as_object=True, fail_silently=True, fix_protocol=True)
-        domain = '.'.join([res.subdomain, res.domain])
-    except Exception:
+
+        if res is not None:
+            domain = '.'.join([res.subdomain, res.domain])
+    except Exception as err:
+        print("[!] Error    : {}".format(
+            colored(err, "red", attrs=["bold"])
+        ))
         pass
 
     score += int(round(entropy.shannon_entropy(domain)*50))
@@ -210,8 +243,23 @@ def score_domain(domain):
 
     return score
 
-if __name__ == "__main__":
+def main():
+    """ """
+    global uagent
+    uagent         = "Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko"
+    global timeout
+    timeout        = args.timeout
+    certstream_url = "wss://certstream.calidog.io"
+    global url_queue
+    url_queue      = Queue.Queue()
+
+    # Print start messages
+    show_summary()
+    show_network(uagent, timeout)
+
+    # Read suspicious.yaml and external.yaml
     with open("suspicious.yaml", "r") as f:
+        global suspicious
         suspicious = yaml.safe_load(f)
 
     with open("external.yaml", "r") as f:
@@ -238,5 +286,58 @@ def score_domain(domain):
             print(colored("At least one extension is required for 'files'.", "red", attrs=["bold"]))
             exit()
 
+    # Start queue and listen for events via Certstream
+    print(colored("Starting queue...\n", "yellow", attrs=["bold"]))
     QueueManager()
+
+    global pbar
+    pbar = tqdm.tqdm(desc="certificate_update", unit="cert")
     certstream.listen_for_events(callback, url=certstream_url)
+
+def show_summary():
+    """Print summary of arguments selected"""
+
+    print("Summary:")
+    print("    timeout : {}".format(args.timeout))
+    print("    tor     : {}\n".format(args.tor))
+    return
+
+def show_network(uagent, timeout):
+    """Select network to use, get IP address, and print message"""
+    global torsocks
+    global proxies
+    if args.tor:
+        ip_type  = "Tor"
+        proxies  = {
+            "http": "socks5h://127.0.0.1:9050",
+            "https": "socks5h://127.0.0.1:9050"
+        }
+        torsocks = "torsocks"
+    else:
+        ip_type  = "Original"
+        proxies  = {}
+        torsocks = ""
+
+    try:
+        global requested_ip
+        requested_ip = requests.get("https://api.ipify.org",
+                                     proxies=proxies,
+                                     headers={"User-Agent": uagent},
+                                     timeout=timeout,
+                                     allow_redirects=True).content
+    except Exception as err:
+        print("[!!] Error   : {}".format(
+            colored(err, "red", attrs=["bold"])
+        ))
+        exit()
+
+    print(colored("Getting IP Address...", "yellow", attrs=["bold"]))
+    if args.tor:
+        obfuscated_ip = ".".join(["XXX.XXX.XXX", requested_ip.split(".")[:-1][0]])
+        print(colored("{} IP: {}\n".format(ip_type, obfuscated_ip), "yellow", attrs=["bold"]))
+    else:
+        print(colored("{} IP: {}\n".format(ip_type, requested_ip), "yellow", attrs=["bold"]))
+    return
+
+if __name__ == "__main__":
+    main()