Merge branch 'optional_directory'

LICENSE

@@ -1,7 +1,7 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 3, 29 June 2007
+                GNU GENERAL PUBLIC LICENSE
+                   Version 3, 29 June 2007
 
- Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -631,8 +631,9 @@ to attach them to the start of each source file to most effectively
 state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
+    The Analyst Arsenal (A²)™ is A tool belt for analysts to continue fighting 
+    the good fight.
+    Copyright (C) 2018  leunammejii
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -652,7 +653,7 @@ Also add information on how to contact you by electronic and paper mail.
   If the program does terminal interaction, make it output a short
 notice like this when it starts in an interactive mode:
 
-    <program>  Copyright (C) <year>  <name of author>
+    Analyst Arsenal (A²)™  Copyright (C) 2018  leunammejii
     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
     This is free software, and you are welcome to redistribute it
     under certain conditions; type `show c' for details.

external.yaml

@@ -25,7 +25,6 @@ files:
     'hta': 'donotcheck'
     'iso': 'application/octet-stream'
     'jar': 'application/java-archive'
-    'json': 'donotcheck'
     'lnk': 'application/octet-stream'
     'ppt': 'application/vnd.ms-powerpoint'
     'ps1': 'donotcheck'

opendir_certstream.py

@@ -7,10 +7,12 @@
 - Recursively download the site when an open directory is found hosting a file with a particular extension
 
 Optional arguments:
-- --quiet   : Don't show wget output
-- --timeout : Set time to wait for a connection
-- --tor     : Download files via the Tor network
-- --verbose : Show error messages
+- --file-dir : Directory to use for interesting files detected
+- --kit-dir  : Directory to use for phishing kits detected
+- --quiet    : Don't show wget output
+- --timeout  : Set time to wait for a connection
+- --tor      : Download files via the Tor network
+- --verbose  : Show error messages
 
 Credit: https://github.com/x0rz/phishing_catcher
 
@@ -22,10 +24,10 @@
 Usage:
 
 ```
-python opendir_certstream.py
+python opendir_certstream.py [--file-dir] [--kit-dir] [--quiet] [--timeout] [--tor] [--verbose]
 ```
 
-Debugger: open("/tmp/splunk_script.txt", "a").write("{}: <MSG>\n".format(<VAR>))
+Debugger: open("/tmp/opendir.txt", "a").write("{}: <MSG>\n".format(<VAR>))
 """
 
 import argparse
@@ -53,6 +55,16 @@
 
 # Parse Arguments
 parser = argparse.ArgumentParser(description="Attempt to detect phishing kits and open directories via Certstream.")
+parser.add_argument("--file-dir",
+                    dest="fdir",
+                    default="./InterestingFile/",
+                    required=False,
+                    help="Directory to use for interesting files detected (default: ./InterestingFiles))")
+parser.add_argument("--kit-dir",
+                    dest="kdir",
+                    default="./KitJackinSeason/",
+                    required=False,
+                    help="Directory to use for phishing kits detected (default: ./KitJackinSeason))")
 parser.add_argument("--quiet",
                     dest="quiet",
                     action="store_true",
@@ -138,12 +150,21 @@ def run(self):
 
                 for ext in extensions:
                     if "{}<".format(ext) in resp.content.lower() and ext in suspicious["archives"]:
-                        directory = "KitJackinSeason"
+                        directory = args.kdir
+
+                        if args.kdir:
+                            directory = args.kdir
                     elif "{}<".format(ext) in resp.content.lower() and ext in suspicious["files"]:
-                        directory = "InterestingFile"
+                        directory = args.fdir
+
+                        if args.fdir:
+                            directory = args.fdir
                     else:
                         continue
 
+                    if not directory.endswith("/"):
+                        directory = "{}/".format(directory)
+
                     tqdm.tqdm.write(
                         "[*] Download  : "
                         "{} ('Index of ' found)".format(
@@ -202,7 +223,8 @@ def callback(message, context):
 
             if score < 75 or \
                  domain.startswith("www.") or \
-                 domain == "chat.kowari.macmoney.co.za" or \
+                 domain.endswith("chat.kowari.macmoney.co.za") or \
+                 domain.endswith("facebook.sitechs.net") or \
                  domain.endswith(".composedb.com") or \
                  domain.endswith(".brilliantpocket.com") or \
                  domain.endswith(".google.com") or \
@@ -348,10 +370,12 @@ def show_summary():
     """Print summary of arguments selected"""
 
     print("Summary:")
-    print("    quiet   : {}".format(args.quiet))
-    print("    timeout : {}".format(args.timeout))
-    print("    tor     : {}".format(args.tor))
-    print("    verbose : {}\n".format(args.verbose))
+    print("    file_dir : {}".format(args.fdir))
+    print("    kit_dir  : {}".format(args.kdir))
+    print("    quiet    : {}".format(args.quiet))
+    print("    timeout  : {}".format(args.timeout))
+    print("    tor      : {}".format(args.tor))
+    print("    verbose  : {}\n".format(args.verbose))
     return
 
 def show_network(uagent, timeout):
@@ -415,7 +439,7 @@ def format_wget(timeout, directory, uagent, url):
         "--no-clobber",
         "--timeout={}".format(timeout),
         "--waitretry=0",
-        "--directory-prefix=./{}/".format(directory),
+        "--directory-prefix={}".format(directory),
         "--header='User-Agent: {}'".format(uagent),
         "--content-disposition",
         "--recursive",

opendir_urlscan.py

@@ -27,10 +27,10 @@
 Usage:
 
 ```
-python opendir_urlscan.py <QUERY_TYPE> <DELTA> <FILE_EXTENSION> [--dry-run] [--exclude=CSV]
+python opendir_urlscan.py <QUERY_TYPE> <DELTA> <FILE_EXTENSION> [--dry-run] [--exclude=CSV] [--quiet] [--timeout] [--tor] [--verbose]
 ```
 
-Debugger: open("/tmp/splunk_script.txt", "a").write("{}: <MSG>\n".format(<VAR>))
+Debugger: open("/tmp/opendir.txt", "a").write("{}: <MSG>\n".format(<VAR>))
 """
 
 import argparse
@@ -73,6 +73,16 @@
                     default="",
                     required=False,
                     help="A comma-separated list of domains to not download content from (ex. 'google.com,bing.com')")
+parser.add_argument("--file-dir",
+                    dest="fdir",
+                    default="./InterestingFile/",
+                    required=False,
+                    help="Directory to use for interesting files detected (default: ./InterestingFiles))")
+parser.add_argument("--kit-dir",
+                    dest="kdir",
+                    default="./KitJackinSeason/",
+                    required=False,
+                    help="Directory to use for phishing kits detected (default: ./KitJackinSeason))")
 parser.add_argument("--quiet",
                     dest="quiet",
                     action="store_true",
@@ -213,14 +223,17 @@ def main():
 
                 for extension in extensions.keys():
                     if ".{}<".format(extension) in resp.content.lower() and extension in archives:
-                        directory = "KitJackinSeason"
+                        directory = args.kdir
                         recursive = True
                     elif ".{}<".format(ext) in resp.content.lower() and extension in files:
-                        directory = "InterestingFile"
+                        directory = args.fdir
                         recursive = False
                     else:
                         continue
 
+                    if not directory.endswith("/"):
+                        directory = "{}/".format(directory)
+
                     print("[*] Download : {} ('Index of ' found)".format(
                         colored(url, "green", attrs=["bold"])
                     ))
@@ -230,8 +243,8 @@ def main():
                         break
 
                     try:
-                        if directory == "InterestingFile":
-                            os.mkdir("./{}/{}".format(directory, domain))
+                        if directory == args.fdir:
+                            os.mkdir("{}{}".format(directory, domain))
 
                         wget_command = format_wget(timeout,
                                                    directory,
@@ -251,10 +264,13 @@ def main():
 
             # A URL is found ending in the specified extension but the server responded with no Content-Type
             if "Content-Type" not in resp.headers.keys():
-                directory = "InterestingFile"
+                directory = args.fdir
                 recursive = False
 
-                if os.path.exists("./{}/{}".format(directory, domain)):
+                if not directory.endswith("/"):
+                    directory = "{}/".format(directory)
+
+                if os.path.exists("{}{}".format(directory, domain)):
                     print("[-] Skipping : {} (Directory '{}' already exists)".format(
                         colored(url, "red"),
                         domain
@@ -276,7 +292,7 @@ def main():
                         break
 
                     try:
-                        os.mkdir("./{}/{}".format(directory, domain))
+                        os.mkdir("{}{}".format(directory, domain))
 
                         wget_command = format_wget(timeout,
                                                    directory,
@@ -296,10 +312,13 @@ def main():
 
             # A file is found with the Mime-Type of the specified extension
             if resp.headers["Content-Type"].startswith(extensions[ext]) or url.endswith(".{}".format(ext)):
-                directory = "InterestingFile"
+                directory = args.fdir
                 recursive = False
 
-                if os.path.exists("./{}/{}".format(directory, domain)):
+                if not directory.endswith("/"):
+                    directory = "{}/".format(directory)
+
+                if os.path.exists("{}{}".format(directory, domain)):
                     print("[-] Skipping : {} (Directory '{}' already exists)".format(
                         colored(url, "red"),
                         domain
@@ -322,7 +341,7 @@ def main():
                     break
 
                 try:
-                    os.mkdir("./{}/{}".format(directory, domain))
+                    os.mkdir("{}{}".format(directory, domain))
 
                     wget_command = format_wget(timeout,
                                                directory,
@@ -352,8 +371,10 @@ def show_summary():
     print("Summary:")
     print("    query_type     : {}".format(args.query_type.lower()))
     print("    delta          : {}".format(args.delta))
-    print("    file_extension : {}".format(args.file_extension.lower()))
     print("    exclusions     : {}".format(args.exclude.split(",")))
+    print("    file_dir       : {}".format(args.fdir))
+    print("    file_extension : {}".format(args.file_extension.lower()))
+    print("    kit_dir        : {}".format(args.kdir))
     print("    quiet          : {}".format(args.quiet))
     print("    timeout        : {}".format(args.timeout))
     print("    tor            : {}".format(args.tor))
@@ -465,7 +486,7 @@ def format_wget(timeout, directory, uagent, recursive, url):
         "--no-clobber",
         "--timeout={}".format(timeout),
         "--waitretry=0",
-        "--directory-prefix=./{}/".format(directory),
+        "--directory-prefix={}".format(directory),
         "--header='User-Agent: {}'".format(uagent),
         "--content-disposition",
         "--no-parent"