diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..41f62d3 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 0000000..19b16c1 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,16 @@ +# file GENERATED by distutils, do NOT edit +README.txt +setup.py +bin/drone-burp +bin/drone-nessus +bin/drone-nexpose +bin/drone-nmap +bin/drone-raw +lairdrone/__init__.py +lairdrone/api.py +lairdrone/drone_models.py +lairdrone/exceptions.py +lairdrone/helper.py +lairdrone/lair_models.py +lairdrone/nmap.py +lairdrone/raw.py diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..b64609e --- /dev/null +++ b/README.txt @@ -0,0 +1,12 @@ +================= +Lair Drones +================= +Lair takes a different approach to uploading, parsing, and ingestion of automated tool output (xml). We push this work off onto client side scripts called drones. These drones connect directly to the database. To use them all you have to do is export an environment variable "MONGO_URL". This variable is probably going to be the same you used for installation + + + export MONGO_URL='mongodb://username:password@ip:27017/lair?ssl=true' + +With the environment variable set you will need a project id to import data. You can grab this from the upper right corner of the lair dashboard next to the project name. You can now run any drones. + + + python nmap.py /path/to/nmap.xml diff --git a/bin/drone-burp b/bin/drone-burp new file mode 100644 index 0000000..1fa5494 --- /dev/null +++ b/bin/drone-burp @@ -0,0 +1,257 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import sys +try: + import xml.etree.ElementTree as et +except ImportError: + print "drone-burp requires the lxml module" + sys.exit(1) + +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) + +from optparse import OptionParser +from urlparse import urlparse +from lairdrone import api, drone_models as models +from lairdrone import helper + +OS_WEIGHT = 75 +TOOL = "burp" + + +def parse(project, burp_file, db, options): + """Parses a Burp file and updates the Lair database + + :param project: The project id + :param burp_file: The Burp xml file to be parsed + :param db: A database connection + """ + try: + import xml.etree.ElementTree as et + except ImportError as e: + print "[!] Error: {0}. Install/upgrade module".format(e) + exit(1) + + tree = et.parse(burp_file) + root = tree.getroot() + + # Create the project dictionary which acts as foundation of document + project_dict = dict(models.project_model) + project_dict['commands'] = list() + project_dict['vulnerabilities'] = list() + project_dict['project_id'] = project + + # Temp dicts used to ensure no duplicate hosts or ports are added + temp_vulns = dict() + temp_hosts = list() + + command_dict = dict(models.command_model) + command_dict['tool'] = TOOL + command_dict['command'] = 'Active scan' + project_dict['commands'].append(command_dict) + + for issue in root.iter('issue'): + + v = dict(models.vulnerability_model) + v['cves'] = list() + v['plugin_ids'] = list() + v['identified_by'] = list() + v['hosts'] = list() + v['notes'] = list() + + # Set CVSS + severity = issue.find('severity') + if severity is not None: + s = severity.text + if s == 'High': + v['cvss'] = 10.0 + # TODO: Confirm literal 'Medium' in XML file + elif s == 'Medium': + v['cvss'] = 5.0 + elif s == 'Low': + v['cvss'] = 3.0 + else: + v['cvss'] = 0.0 + + # Set title + name = issue.find('name') + if name is not None: + v['title'] = name.text + + # Set plugin + plugin_elem = issue.find('type') + if plugin_elem is not None: + plugin_id = plugin_elem.text + + plugin_dict = dict(models.plugin_id_model) + plugin_dict['tool'] = TOOL + plugin_dict['id'] = plugin_id + v['plugin_ids'].append(plugin_dict) + + # Set identified by information + identified_dict = dict(models.identified_by_model) + identified_dict['tool'] = TOOL + identified_dict['id'] = plugin_id + v['identified_by'].append(identified_dict) + + # Search for solution + solution = issue.find('remediationBackground') + if solution is not None: + v['solution'] = solution.text + + # Search for description + description = issue.find('issueBackground') + if description is not None: + v['description'] = description.text + + # Search for evidence + evidence = issue.find('issueDetail') + if evidence is not None: + v['evidence'] = evidence.text + evidence = evidence.text + + if plugin_id not in temp_vulns: + # New vulnerability not already identified + # By default, don't include informational findings unless + # explicitly told to do so. + if v['cvss'] == 0 and not options.include_informational: + continue + temp_vulns[plugin_id] = v + + host_dict = dict(models.host_model) + host_dict['os'] = list() + host_dict['ports'] = list() + host_dict['hostnames'] = list() + + # Used later for adding a note to specify location of vulnerability + # on the server. + path = '' + path_elem = issue.find('path') + if path_elem is not None: + path = path_elem.text + + # Search for host element + host_elem = issue.find('host') + if host_elem is not None: + string_addr = host_elem.attrib['ip'] + # Occasionally, host name is present but no IP address. Ignore + # those findings. + if not string_addr: + continue + + long_addr = helper.ip2long(string_addr) + host_dict['string_addr'] = string_addr + host_dict['long_addr'] = long_addr + + # Determine port + port_dict = dict(models.port_model) + port_dict['port'] = 80 + port_dict['protocol'] = models.PROTOCOL_TCP + + result = urlparse(host_elem.text) + + # Determine if service is http or https + if result.port: + port_dict['port'] = result.port + else: + if result.scheme == 'https': + port_dict['port'] = 443 + + if result.scheme == 'https': + port_dict['service'] = 'https' + else: + port_dict['service'] = 'http' + + # Grab the hostname + if result.hostname: + host_dict['hostnames'].append(result.hostname) + + # Add a note regarding vulnerable path + if path: + content = host_elem.text + path + + # Include evidence in the note as many times this includes + # specific paramaters and test strings that ultimately will + # make it easier to validate from the UI + if evidence: + content += "\n" + evidence + + note_dict = dict(models.note_model) + note_dict['title'] = 'Details' + note_dict['content'] = content + v['notes'].append(note_dict) + + host_dict['ports'].append(port_dict) + + # Don't set an OS + os_dict = dict(models.os_model) + os_dict['tool'] = TOOL + host_dict['os'].append(os_dict) + + # Check if host/port is already associated with project. + found_host = False + for h in temp_hosts: + if h['string_addr'] == string_addr: + found_host = True + + found_port = False + for p in h['ports']: + if p['port'] == port_dict['port']: + # Do nothing as host/port is in the list already + found_port = True + + # Did not find port, add it + if not found_port: + h['ports'].append(port_dict) + + # Host has not yet been encountered + if not found_host: + temp_hosts.append(host_dict) + + host_key_dict = dict(models.host_key_model) + host_key_dict['string_addr'] = string_addr + host_key_dict['port'] = port_dict['port'] + + # Check if host/port is already associated with vuln, add if not + if plugin_id in temp_vulns and \ + host_key_dict not in temp_vulns[plugin_id]['hosts']: + temp_vulns[plugin_id]['hosts'].append(host_key_dict) + + project_dict['vulnerabilities'] = temp_vulns.values() + project_dict['hosts'] = temp_hosts + + return project_dict + +if __name__ == '__main__': + + usage = "usage: %prog " + description = "%prog imports Burp files into Lair" + + parser = OptionParser(usage=usage, description=description, + version="%prog 0.0.1") + parser.add_option( + "--include-informational", + dest="include_informational", + default=False, + action="store_true", + help="Forces informational plugins to be loaded" + ) + + (options, args) = parser.parse_args() + + if len(args) != 2: + print parser.get_usage() + exit(1) + + # Connect to the database + db = api.db_connect() + + project = parse(args[0], args[1], db, options) + + api.save(project, db, TOOL) + + exit(0) diff --git a/bin/drone-nessus b/bin/drone-nessus new file mode 100644 index 0000000..722e020 --- /dev/null +++ b/bin/drone-nessus @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import sys +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) + +from optparse import OptionParser +from lairdrone import api +from lairdrone import nessus + + +if __name__ == '__main__': + + usage = "usage: %prog " + description = "%prog imports Nessus files into Lair" + + parser = OptionParser(usage=usage, description=description, + version="%prog 0.0.4") + parser.add_option( + "--include-informational", + dest="include_informational", + default=False, + action="store_true", + help="Forces informational plugins to be loaded" + ) + + parser.add_option( + "--min-note-severity", + dest="min_note_severity", + default=2, + action="store", + type="int", + help="Minimal severity level to use when persisting service notes " + "(range 0-4, default 2)" + ) + + (options, args) = parser.parse_args() + + if len(args) != 2: + print parser.get_usage() + exit(1) + + if options.min_note_severity < 0 or options.min_note_severity > 4: + print parser.get_usage() + sys.exit(1) + + # Connect to the database + db = api.db_connect() + + project = nessus.parse(args[0], args[1], options) + api.save(project, db, nessus.TOOL) + exit(0) diff --git a/bin/drone-nexpose b/bin/drone-nexpose new file mode 100644 index 0000000..d63a8ac --- /dev/null +++ b/bin/drone-nexpose @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import sys +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) + +from optparse import OptionParser +from lairdrone import api +from lairdrone import nexpose + + +if __name__ == '__main__': + + usage = "usage: %prog " + description = "%prog imports Nexpose files into Lair" + + parser = OptionParser(usage=usage, description=description, + version="%prog 0.0.2") + parser.add_option( + "--include-informational", + dest="include_informational", + default=False, + action="store_true", + help="Forces informational plugins to be loaded" + ) + + (options, args) = parser.parse_args() + + if len(args) != 2: + print parser.get_usage() + exit(1) + + # Connect to the database + db = api.db_connect() + + project = nexpose.parse(args[0], args[1], options) + + api.save(project, db, nexpose.TOOL) + + exit(0) diff --git a/bin/drone-nmap b/bin/drone-nmap new file mode 100644 index 0000000..7625912 --- /dev/null +++ b/bin/drone-nmap @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import sys +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) +from optparse import OptionParser +from lairdrone import api +from lairdrone import nmap + + +def main(): + """ + main point of execution + + :return: + """ + + usage = "usage: %prog " + description = "%prog imports Nmap files into Lair" + parser = OptionParser(usage=usage, description=description, + version="%prog 0.0.1") + (options, args) = parser.parse_args() + if len(args) != 2: + print parser.get_usage() + sys.exit(1) + # Connect to the database + db = api.db_connect() + + project = nmap.parse(args[0], args[1]) + api.save(project, db, nmap.TOOL) + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/bin/drone-raw b/bin/drone-raw new file mode 100644 index 0000000..b921a8a --- /dev/null +++ b/bin/drone-raw @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import sys +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) + +from optparse import OptionParser +from lairdrone import api + + +if __name__ == '__main__': + + usage = "usage: %prog " + description = "%prog imports raw JSON files into Lair" + + parser = OptionParser(usage=usage, description=description, + version="%prog 0.0.1") + + (options, args) = parser.parse_args() + + if len(args) != 3: + print parser.get_usage() + exit(1) + + # connect to database + db = api.db_connect() + + from lairdrone import raw + project = raw.parse(args[0], args[1]) + + api.save(project, db, args[2]) + + exit(0) + diff --git a/lairdrone/__init__.py b/lairdrone/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lairdrone/api.py b/lairdrone/api.py new file mode 100644 index 0000000..454035f --- /dev/null +++ b/lairdrone/api.py @@ -0,0 +1,354 @@ +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +__author__ = 'Dan Kottmann ' + +import os +import copy +import hashlib +from pymongo import ASCENDING, DESCENDING +from datetime import datetime +from bson.objectid import ObjectId +from exceptions import MissingRequiredSchemaField, ProjectDoesNotExistError, \ + IncompatibleVersionError +import lair_models + +DRONE_LOG_HISTORY = 500 + +# this is the document version +# only serious changes to the lair api will update this +VERSION = '0.1.0' + + +def db_connect(): + """ + connect to the database + + :return:database connection object + """ + from pymongo import Connection, uri_parser + + # Connect to the database + if 'MONGO_URL' not in os.environ: + print "[***] Missing 'MONGO_URL' Environment Variable [***]" + raise EnvironmentError + + mongo_options = uri_parser.parse_uri(os.environ['MONGO_URL']) + (host, port) = mongo_options['nodelist'][0] + + ssl = mongo_options['options'].get('ssl', False) + + print "[+] Attempting connection to database '{0}:{1}/{2}'".format( + host, str(port), mongo_options['database'] + ) + conn = Connection(host, port, ssl=ssl) + db = conn[mongo_options['database']] + + if mongo_options['username'] or mongo_options['password']: + db.authenticate(mongo_options['username'], mongo_options['password']) + + print "[+] Connection successful." + + return db + + +def validate(document): + + """Check that the document schema is valid + + :param document: dictionary to validate + """ + + # Validate project_id + if 'project_id' not in document or not document['project_id']: + raise MissingRequiredSchemaField('project_id') + + # Validate command + if 'commands' not in document or not document['commands']: + raise MissingRequiredSchemaField('commands') + + return True + + +def save(document, db, tool): + """Save the project details in the Lair database. + + :param document: A complete representation of the project model + :param db: A connection to the target Lair database + :raise: MissingRequiredSchemaField, ProjectDoesNotExistError + """ + + # Validate compatible versions + version = db.versions.find_one() + if version['version'] != VERSION: + raise IncompatibleVersionError(VERSION, version['version']) + + # Validate the schema - will raise an error if invalid + validate(document) + + print "[+] Processing project {0}".format(document['project_id']) + + temp_drone_log = list() + + q = {'_id': document['project_id']} + + # Ensure the project exists in the database + if db.projects.find(q).count() != 1: + raise ProjectDoesNotExistError(document['project_id']) + + project = db.projects.find_one(q) + + # Add the command + project['commands'].extend(document['commands']) + + # Add project notes + project['notes'].extend(document['notes']) + + # Add the owner if it isn't already set + if 'owner' not in project or not project['owner']: + project['owner'] = document['owner'] + + # Add the industry if not already set + if 'industry' not in project or not project['industry']: + project['industry'] = document.get('industry', 'N/A') + + # Add the creation date if not already set + if 'creation_date' not in project or not project['creation_date']: + project['creation_date'] = document['creation_date'] + + # Add the description if not already set + if 'description' not in project or not project['description']: + project['description'] = document.get('description', '') + + if 'hosts' not in project: + project['hosts'] = list() + + if 'vulnerabilities' not in project: + project['vulnerabilities'] = list() + + if len(project['vulnerabilities']) == 0 and len(project['hosts']) == 0: + now = datetime.utcnow().isoformat() + temp_drone_log.append("{0} - Initial project load".format(now)) + + # Create indexes + db.hosts.ensure_index([ + ('project_id', ASCENDING), + ('string_addr', ASCENDING) + ]) + db.ports.ensure_index([ + ('project_id', ASCENDING), + ('host_id', ASCENDING), + ('port', ASCENDING), + ('protocol', ASCENDING) + ]) + db.vulnerabilities.ensure_index([ + ('project_id', ASCENDING), + ('plugin_ids', ASCENDING) + ]) + + # For each host in the parsed scan, check to see if it already + # exists in the database. + for file_host in document['hosts']: + + is_known_host = True + host = db.hosts.find_one({'project_id': project['_id'], 'string_addr': file_host['string_addr']}) + if not host: + is_known_host = False + host = copy.deepcopy(lair_models.host_model) + + pre_md5 = hashlib.md5() + pre_md5.update(str(host)) + + host['project_id'] = project['_id'] + host['alive'] = file_host['alive'] + host['string_addr' ] = file_host['string_addr'] + host['long_addr'] = file_host['long_addr'] + host['is_profiled'] = file_host.get('is_profiled', False) + host['is_enumerated'] = file_host.get('is_enumerated', False) + + # Add hostnames + if len(file_host['hostnames']) > 0: + # Only update if new host names were identified + if not set(file_host['hostnames']).issubset(host['hostnames']): + host['hostnames'].extend(file_host['hostnames']) + host['hostnames'] = list(set(host['hostnames'])) + host['last_modified_by'] = tool + + # Update MAC address if it's not set already + if not host['mac_addr']: + host['mac_addr'] = file_host['mac_addr'] + + # Add the operating system + if file_host['os']: + os_list = [] + # The following ensures that no duplicate entries are + # added to the database. + for file_os in file_host['os']: + dupe_found = False + for db_os in host['os']: + if db_os['tool'] == file_os['tool'] and \ + db_os['fingerprint'] == \ + file_os['fingerprint']: + dupe_found = True + + if not dupe_found: + os_list.append(file_os) + host['last_modified_by'] = tool + + host['os'].extend(os_list) + + post_md5 = hashlib.md5() + post_md5.update(str(host)) + + # Only save if changes were detected + if pre_md5 != post_md5: + host['last_modified_by'] = tool + if not is_known_host: + id = str(ObjectId()) + host['_id'] = id + host['status'] = lair_models.STATUS_GREY + + db.hosts.save(host) + + if not is_known_host: + now = datetime.utcnow().isoformat() + temp_drone_log.append("{0} - New host found: {1}".format( + now, + file_host['string_addr']) + ) + + # Process each port for the host, checking against known ports + for file_port in file_host['ports']: + + q = { + 'project_id': project['_id'], + 'host_id': host['_id'], + 'port': file_port['port'], + 'protocol': file_port['protocol'] + } + port = db.ports.find_one(q) + + is_known_port = False + if port: + is_known_port = True + else: + port = copy.deepcopy(lair_models.port_model) + + pre_md5 = hashlib.md5() + pre_md5.update(str(port)) + + port['host_id'] = host['_id'] + port['project_id'] = project['_id'] + port['protocol'] = file_port['protocol'] + port['port'] = file_port['port'] + + # TODO: Determine how to handle a closed port + port['alive'] = file_port['alive'] + + # Update product if it is unknown + if port['product'] == lair_models.PRODUCT_UNKNOWN: + port['product'] = file_port['product'] + + # Set the service if it is not set + if not port['service'] or port['service'] == 'unknown': + port['service'] = file_port['service'] + + # Include any script output for the port + if file_port['notes']: + port['notes'].extend(file_port['notes']) + + if not is_known_port: + id = str(ObjectId()) + port['_id'] = id + port['status'] = lair_models.STATUS_GREY + now = datetime.utcnow().isoformat() + temp_drone_log.append("{0} - New port found: {1}/{2} ({3})".format( + now, + str(file_port['port']), + file_port['protocol'], + file_port['service']) + ) + + post_md5 = hashlib.md5() + post_md5.update(str(port)) + + if pre_md5 != post_md5: + port['last_modified_by'] = tool + db.ports.save(port) + + # For each vulnerability in the parsed scan, check to see if it already + # exists in the database. + for file_vuln in document.get('vulnerabilities', []): + + is_known_vuln = False + + # Attempt a lookup by plugin_id... + q = { + 'project_id': project['_id'], + 'plugin_ids': {'$all': file_vuln['plugin_ids']} + } + db_vuln = db.vulnerabilities.find_one(q) + + if db_vuln: + is_known_vuln = True + + # No vuln found by plugin_id, treat as new + if not is_known_vuln: + db_vuln = copy.deepcopy(file_vuln) + id = str(ObjectId()) + db_vuln['status'] = lair_models.STATUS_GREY + db_vuln['_id'] = id + db_vuln['project_id'] = project['_id'] + db_vuln['last_modified_by'] = tool + now = datetime.utcnow().isoformat() + temp_drone_log.append("{0} - New vulnerability found: {1}".format( + now, + file_vuln['title']) + ) + db.vulnerabilities.save(db_vuln) + + if is_known_vuln: + pre_md5 = hashlib.md5() + pre_md5.update(str(db_vuln)) + + db_vuln['cves'].extend(file_vuln['cves']) + db_vuln['cves'] = list(set(db_vuln['cves'])) + db_vuln['identified_by'].extend(file_vuln['identified_by']) + + # Only set 'flag' if it's true for parsed vuln + db_vuln['flag'] = file_vuln['flag'] \ + if file_vuln.get('flag', False) else db_vuln.get('flag', False) + + # Include any script output for the port + if file_vuln['notes']: + db_vuln['notes'].extend(file_vuln['notes']) + + for file_host in file_vuln['hosts']: + if file_host not in db_vuln['hosts']: + db_vuln['hosts'].append(file_host) + now = datetime.utcnow().isoformat() + temp_drone_log.append("{0} - {1}:{2}/{3} - New vulnerability found: {4}".format( + now, + file_host['string_addr'], + str(file_host['port']), + file_host['protocol'], + file_vuln['title']) + ) + + post_md5 = hashlib.md5() + post_md5.update(str(db_vuln)) + + # Vulnerability was known, but change was detected + if pre_md5 != post_md5: + db_vuln['last_modified_by'] = tool + db.vulnerabilities.save(db_vuln) + + # Ensure history log does not exceed DRONE_LOG_HISTORY limit + project['drone_log'].extend(temp_drone_log) + length = len(project['drone_log']) + del project['drone_log'][0:(length - DRONE_LOG_HISTORY)] + + db.projects.save(project) + + print "[+] Processing completed: {0} host(s) processed.".format( + str(len(document['hosts']))) diff --git a/lairdrone/drone_models.py b/lairdrone/drone_models.py new file mode 100644 index 0000000..ec85b2d --- /dev/null +++ b/lairdrone/drone_models.py @@ -0,0 +1,116 @@ +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +# Constants to be utilized by clients for consistency +STATUS_CLEAN = 'clean' +STATUS_EXPLOITED = 'exploited' +STATUS_IN_PROGRESS = 'in_progress' +STATUS_UNDETERMINED = 'undetermined' + +PROTOCOL_TCP = 'tcp' +PROTOCOL_UDP = 'udp' + +PRODUCT_UNKNOWN = 'unknown' + +# Model definitions +command_model = { + 'tool': '', + 'command': '' +} + +os_model = { + 'tool': '', + 'weight': 0, + 'fingerprint': 'unknown' +} + +credential_model = { + 'username': '', + 'password': '', + 'hash': '' +} + +note_model = { + 'title': '', + 'content': '', + 'last_modified_by': '' +} + +port_model = { + 'port': 0, + 'protocol': PROTOCOL_TCP, + 'service': '', + 'product': PRODUCT_UNKNOWN, + 'alive': True, + 'status': STATUS_UNDETERMINED, + 'credentials': [], # credential_models + 'notes': [], # note_models + 'last_modified_by': '' +} + +host_model = { + 'long_addr': 0, + 'string_addr': '', + 'mac_addr': '', + 'hostnames': [], # Strings + 'os': [], # os_models + 'alive': True, + 'status': STATUS_UNDETERMINED, + 'ports': [], # port_models + 'last_modified_by': '', + 'notes': [] +} + +plugin_id_model = { + 'tool': '', + 'id': '' +} + +identified_by_model = { + 'tool': '', + 'id': '' +} + +host_key_model = { + 'string_addr': '', + 'port': 0, + 'protocol': PROTOCOL_TCP +} + +vulnerability_model = { + 'title': '', + 'description': '', + 'solution': '', + 'status': STATUS_UNDETERMINED, + 'cvss': 0, + 'cves': [], # Strings + 'plugin_ids': [], # plugin_id_models + 'identified_by': [], # identified_by_models + 'confirmed': False, + 'flag': False, + 'notes': [], # note_models + 'evidence': '', + 'hosts': [], # host_key_model + 'last_modified_by': '' +} + +attack_model = { + 'title': '', + 'content': '' +} + +project_model = { + 'project_id': '', + 'project_name': '', + 'industry': '', + 'creation_date': '', + 'description': '', + 'owner': '', + 'contributors': [], # ObjectIds + 'commands': [], # command_models + 'notes': [], # note_models + 'hosts': [], # host_models + 'vulnerabilities': [], # vulnerability_models + 'attacks': [], # attack_models + 'drone_log': [] +} diff --git a/lairdrone/exceptions.py b/lairdrone/exceptions.py new file mode 100644 index 0000000..c94a5c2 --- /dev/null +++ b/lairdrone/exceptions.py @@ -0,0 +1,47 @@ +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +__author__ = 'Dan Kottmann ' + + +class MissingRequiredSchemaField(Exception): + + def __init__(self, field): + self.field = str(field) + + def __str__(self): + return "A required field is missing from the " \ + "dictionary: {0}".format(repr(self.field)) + + +class ProjectDoesNotExistError(Exception): + + def __init__(self, project): + self.project = str(project) + + def __str__(self): + return "Project does not exist in the database: {0}.".format( + repr(self.project)) + + +class IncompatibleVersionError(Exception): + + def __init__(self, version, curr_version): + self.version = version + self.curr_version = curr_version + + def __str__(self): + return "Incompatible API version found: v{0}. Upgrade to v{1}.".format( + self.version, + self.curr_version + ) + + +class IncompatibleDataVersionError(Exception): + + def __init__(self, version): + self.version = version + + def __str__(self): + return "The input file is not a supported version. Expecting " \ + "{0}.".format(self.version) diff --git a/lairdrone/helper.py b/lairdrone/helper.py new file mode 100644 index 0000000..b9aa58d --- /dev/null +++ b/lairdrone/helper.py @@ -0,0 +1,23 @@ +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import socket +import struct + + +def ip2long(ip): + """Calculate the IP address as a long integer + + :param ip: IP address as a dotted-quad string + :return: Long value of the IP address + """ + return struct.unpack('!L', socket.inet_aton(ip))[0] + + +def long2ip(n): + """Calculate the dotted-quad string representation of a long integer + + :param n: IP address as a long value + :return: IP address as a dotted-quad string + """ + return socket.inet_ntoa(struct.pack('!L', n)) diff --git a/lairdrone/lair_models.py b/lairdrone/lair_models.py new file mode 100644 index 0000000..b1deda1 --- /dev/null +++ b/lairdrone/lair_models.py @@ -0,0 +1,110 @@ +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +# Constants to be utilized by clients for consistency +VERSION = '0.1.0' +STATUS_GREY = 'lair-grey' +STATUS_BLUE = 'lair-blue' +STATUS_GREEN = 'lair-green' +STATUS_ORANGE = 'lair-orange' +STATUS_RED = 'lair-red' +STATUS_MAP = [STATUS_GREY, STATUS_BLUE, STATUS_GREEN, STATUS_ORANGE, STATUS_RED] +PROTOCOL_TCP = 'tcp' +PROTOCOL_UDP = 'udp' +PRODUCT_UNKNOWN = 'unknown' +SERVICE_UNKNOWN = 'unknown' + +# Model definitions +command_model = { + 'tool': '', + 'command': '' +} + +os_model = { + 'tool': '', + 'weight': 0, + 'fingerprint': 'unknown' +} + +credential_model = { + 'username': '', + 'password': '' +} + +note_model = { + 'title': '', + 'content': '', + 'last_modified_by': '' +} + +port_model = { + 'project_id': '', + 'host_id': '', + 'port': 0, + 'protocol': PROTOCOL_TCP, + 'service': SERVICE_UNKNOWN, + 'product': PRODUCT_UNKNOWN, + 'alive': True, + 'status': STATUS_GREY, + 'credentials': [], # credential_models + 'notes': [], # note_models + 'last_modified_by': '' +} + +host_model = { + 'project_id': '', + 'long_addr': 0, + 'string_addr': '', + 'mac_addr': '', + 'hostnames': [], # Strings + 'os': [], # os_models + 'notes': [], + 'alive': True, + 'status': STATUS_GREY, + 'last_modified_by': '' +} + +plugin_id_model = { + 'tool': '', + 'id': '' +} + +identified_by_model = { + 'tool': '', + 'id': '' +} + +host_key_model = { + 'string_addr': '', + 'port': 0, + 'protocol': PROTOCOL_TCP +} + +vulnerability_model = { + 'project_id': '', + 'title': '', + 'description': '', + 'solution': '', + 'status': STATUS_GREY, + 'cvss': 0, + 'cves': [], # Strings + 'plugin_ids': [], # plugin_id_models + 'identified_by': [], # identified_by_models + 'confirmed': False, + 'flag': False, + 'notes': [], # note_models + 'evidence': '', + 'hosts': [], # host_key_model + 'last_modified_by': '' +} + +project_model = { + 'project_name': '', + 'owner': '', + 'contributors': [], # ObjectIds + 'commands': [], # command_models + 'notes': [], # note_models + 'drone_log': [], + 'messages': [], # attack_models + 'files': [] +} \ No newline at end of file diff --git a/lairdrone/nessus.py b/lairdrone/nessus.py new file mode 100644 index 0000000..9bca608 --- /dev/null +++ b/lairdrone/nessus.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import xml.etree.ElementTree as et +import re +import copy +from lairdrone import drone_models as models +from lairdrone import helper + +OS_WEIGHT = 75 +TOOL = "nessus" + + +def parse(project, nessus_file, include_informational=False, min_note_sev=2): + """Parses a Nessus XMLv2 file and updates the Hive database + + :param project: The project id + :param nessus_file: The Nessus xml file to be parsed + :param include_informational: Whether to include info findings in data. Default False + :min_note_sev: The minimum severity of notes that will be saved. Default 2 + """ + + cve_pattern = re.compile(r'(CVE-|CAN-)') + false_udp_pattern = re.compile(r'.*\?$') + + tree = et.parse(nessus_file) + root = tree.getroot() + note_id = 1 + + # Create the project dictionary which acts as foundation of document + project_dict = dict(models.project_model) + project_dict['commands'] = list() + project_dict['vulnerabilities'] = list() + project_dict['project_id'] = project + + # Used to maintain a running list of host:port vulnerabilities by plugin + vuln_host_map = dict() + + for host in root.iter('ReportHost'): + + temp_ip = host.attrib['name'] + + host_dict = dict(models.host_model) + host_dict['os'] = list() + host_dict['ports'] = list() + host_dict['hostnames'] = list() + + # Tags contain host-specific information + for tag in host.iter('tag'): + + # Operating system tag + if tag.attrib['name'] == 'operating-system': + os_dict = dict(models.os_model) + os_dict['tool'] = TOOL + os_dict['weight'] = OS_WEIGHT + os_dict['fingerprint'] = tag.text + host_dict['os'].append(os_dict) + + # IP address tag + if tag.attrib['name'] == 'host-ip': + host_dict['string_addr'] = tag.text + host_dict['long_addr'] = helper.ip2long(tag.text) + + # MAC address tag + if tag.attrib['name'] == 'mac-address': + host_dict['mac_addr'] = tag.text + + # Hostname tag + if tag.attrib['name'] == 'host-fqdn': + host_dict['hostnames'].append(tag.text) + + # NetBIOS name tag + if tag.attrib['name'] == 'netbios-name': + host_dict['hostnames'].append(tag.text) + + # Track the unique port/protocol combos for a host so we don't + # add duplicate entries + ports_processed = dict() + + # Process each 'ReportItem' + for item in host.findall('ReportItem'): + plugin_id = item.attrib['pluginID'] + plugin_family = item.attrib['pluginFamily'] + severity = int(item.attrib['severity']) + title = item.attrib['pluginName'] + + port = int(item.attrib['port']) + protocol = item.attrib['protocol'] + service = item.attrib['svc_name'] + evidence = item.find('plugin_output') + + # Ignore false positive UDP services + if protocol == "udp" and false_udp_pattern.match(service): + continue + + # Create a port model and temporarily store it in the dict + # for tracking purposes. The ports_processed dict is used + # later to add ports to the host so that no duplicates are + # present. This is necessary due to the format of the Nessus + # XML files. + if '{0}:{1}'.format(port, protocol) not in ports_processed: + port_dict = copy.deepcopy(models.port_model) + port_dict['port'] = port + port_dict['protocol'] = protocol + port_dict['service'] = service + ports_processed['{0}:{1}'.format(port, protocol)] = port_dict + + # Set the evidence as a port note if it exists + if evidence is not None and \ + severity >= min_note_sev and \ + plugin_family != 'Port scanners' and \ + plugin_family != 'Service detection': + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = "{0} (ID{1})".format(title, str(note_id)) + e = evidence.text.strip() + for line in e.split("\n"): + line = line.strip() + if line: + note_dict['content'] += " " + line + "\n" + note_dict['last_modified_by'] = TOOL + ports_processed['{0}:{1}'.format(port, protocol)]['notes'].append(note_dict) + note_id += 1 + + # This plugin is general scan info...use it for 'command' element + if plugin_id == '19506': + + command = item.find('plugin_output') + + command_dict = dict(models.command_model) + command_dict['tool'] = TOOL + + if command is not None: + command_dict['command'] = command.text + + if not project_dict['commands']: + project_dict['commands'].append(command_dict) + + continue + + # Check if this vulnerability has been seen in this file for + # another host. If not, create a new vulnerability_model and + # maintain a mapping between plugin-id and vulnerability as + # well as a mapping between plugin-id and host. These mappings + # are later used to completed the Hive schema such that host + # IP and port information are embedded within each vulnerability + # while ensuring no duplicate data exists. + if plugin_id not in vuln_host_map: + + v = copy.deepcopy(models.vulnerability_model) + v['cves'] = list() + v['plugin_ids'] = list() + v['identified_by'] = list() + v['hosts'] = list() + + # Set the title + v['title'] = title + + # Set the description + description = item.find('description') + if description is not None: + v['description'] = description.text + + # Set the solution + solution = item.find('solution') + if solution is not None: + v['solution'] = solution.text + + # Set the evidence + if evidence is not None: + v['evidence'] = evidence.text + + # Set the vulnerability flag if exploit exists + exploit = item.find('exploit_available') + if exploit is not None: + v['flag'] = exploit.text == 'true' + + # Grab Metasploit details + exploit_detail = item.find('exploit_framework_metasploit') + if exploit_detail is not None and \ + exploit_detail.text == 'true': + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = 'Metasploit Exploit' + note_dict['content'] = 'Exploit exists. Details unknown.' + module = item.find('metasploit_name') + if module is not None: + note_dict['content'] = module.text + note_dict['last_modified_by'] = TOOL + v['notes'].append(note_dict) + + # Grab Canvas details + exploit_detail = item.find('exploit_framework_canvas') + if exploit_detail is not None and \ + exploit_detail.text == 'true': + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = 'Canvas Exploit' + note_dict['content'] = 'Exploit exists. Details unknown.' + module = item.find('canvas_package') + if module is not None: + note_dict['content'] = module.text + note_dict['last_modified_by'] = TOOL + v['notes'].append(note_dict) + + # Grab Core Impact details + exploit_detail = item.find('exploit_framework_core') + if exploit_detail is not None and \ + exploit_detail.text == 'true': + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = 'Core Impact Exploit' + note_dict['content'] = 'Exploit exists. Details unknown.' + module = item.find('core_name') + if module is not None: + note_dict['content'] = module.text + note_dict['last_modified_by'] = TOOL + v['notes'].append(note_dict) + + # Grab ExploitHub SKUs + exploit_detail = item.find('exploit_framework_exploithub') + if exploit_detail is not None and \ + exploit_detail.text == 'true': + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = 'Exploit Hub Exploit' + note_dict['content'] = 'Exploit exists. Details unknown.' + module = item.find('exploithub_sku') + if module is not None: + note_dict['content'] = module.text + note_dict['last_modified_by'] = TOOL + v['notes'].append(note_dict) + + # Grab any and all ExploitDB IDs + details = item.iter('edb-id') + if details is not None: + for module in details: + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = 'Exploit-DB Exploit ' \ + '({0})'.format(module.text) + note_dict['content'] = module.text + note_dict['last_modified_by'] = TOOL + v['notes'].append(note_dict) + + # Set the CVSS score + cvss = item.find('cvss_base_score') + if cvss is not None: + v['cvss'] = float(cvss.text) + else: + risk_factor = item.find('risk_factor') + if risk_factor is not None: + rf = risk_factor.text + if rf == "Low": + v['cvss'] = 3.0 + elif rf == "Medium": + v['cvss'] = 5.0 + elif rf == "High": + v['cvss'] = 7.5 + elif rf == "Critical": + v['cvss'] = 10.0 + + # Set the CVE(s) + for cve in item.findall('cve'): + c = cve_pattern.sub('', cve.text) + v['cves'].append(c) + + # Set the plugin information + plugin_dict = dict(models.plugin_id_model) + plugin_dict['tool'] = TOOL + plugin_dict['id'] = plugin_id + v['plugin_ids'].append(plugin_dict) + + # Set the identified by information + identified_dict = dict(models.identified_by_model) + identified_dict['tool'] = TOOL + identified_dict['id'] = plugin_id + v['identified_by'].append(identified_dict) + + # By default, don't include informational findings unless + # explicitly told to do so. + if v['cvss'] == 0 and not include_informational: + continue + + vuln_host_map[plugin_id] = dict() + vuln_host_map[plugin_id]['hosts'] = set() + vuln_host_map[plugin_id]['vuln'] = v + + if plugin_id in vuln_host_map: + vuln_host_map[plugin_id]['hosts'].add( + "{0}:{1}:{2}".format( + host_dict['string_addr'], + str(port), + protocol + ) + ) + + # In the event no IP was found, use the 'name' attribute of + # the 'ReportHost' element + if not host_dict['string_addr']: + host_dict['string_addr'] = temp_ip + host_dict['long_addr'] = helper.ip2long(temp_ip) + + # Add all encountered ports to the host + host_dict['ports'].extend(ports_processed.values()) + + project_dict['hosts'].append(host_dict) + + # This code block uses the plugin/host/vuln mapping to associate + # all vulnerable hosts to their vulnerability data within the + # context of the expected Hive schema structure. + for plugin_id, data in vuln_host_map.items(): + + # Build list of host and ports affected by vulnerability and + # assign that list to the vulnerability model + for key in data['hosts']: + (string_addr, port, protocol) = key.split(':') + + host_key_dict = dict(models.host_key_model) + host_key_dict['string_addr'] = string_addr + host_key_dict['port'] = int(port) + host_key_dict['protocol'] = protocol + data['vuln']['hosts'].append(host_key_dict) + + project_dict['vulnerabilities'].append(data['vuln']) + + if not project_dict['commands']: + # Adds a dummy 'command' in the event the the Nessus plugin used + # to populate the data was not run. The Lair API expects it to + # contain a value. + command = copy.deepcopy(models.command_model) + command['tool'] = TOOL + command['command'] = "Nessus scan - command unknown" + project_dict['commands'].append(command) + + return project_dict \ No newline at end of file diff --git a/lairdrone/nexpose.py b/lairdrone/nexpose.py new file mode 100644 index 0000000..8b29397 --- /dev/null +++ b/lairdrone/nexpose.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import xml.etree.ElementTree as et +import os +import sys +import re +import copy +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) +from lairdrone import drone_models as models +from lairdrone import helper +from lairdrone.exceptions import IncompatibleDataVersionError + +OS_WEIGHT = 75 +TOOL = "nexpose" + + +def parse(project, nexpose_file, include_informational=False): + """Parses a Nexpose XMLv2 file and updates the Lair database + + :param project: The project id + :param nexpose_file: The Nexpose xml file to be parsed + :include_informational: Whether to include info findings in data. Default False + """ + + cve_pattern = re.compile(r'(CVE-|CAN-)') + html_tag_pattern = re.compile(r'<.*?>') + white_space_pattern = re.compile(r'\s+', re.MULTILINE) + + # Used to create unique notes in DB + note_id = 1 + + tree = et.parse(nexpose_file) + root = tree.getroot() + if root is None or \ + root.tag != "NexposeReport" or \ + root.attrib['version'] != "2.0": + raise IncompatibleDataVersionError("Nexpose XML 2.0") + + # Create the project dictionary which acts as foundation of document + project_dict = dict(models.project_model) + project_dict['commands'] = list() + project_dict['vulnerabilities'] = list() + project_dict['project_id'] = project + project_dict['commands'].append({'tool': TOOL, 'command': 'scan'}) + + # Used to maintain a running list of host:port vulnerabilities by plugin + vuln_host_map = dict() + + for vuln in root.iter('vulnerability'): + v = copy.deepcopy(models.vulnerability_model) + v['cves'] = list() + v['plugin_ids'] = list() + v['identified_by'] = list() + v['hosts'] = list() + + v['cvss'] = float(vuln.attrib['cvssScore']) + v['title'] = vuln.attrib['title'] + plugin_id = vuln.attrib['id'].lower() + + # Set plugin id + plugin_dict = dict(models.plugin_id_model) + plugin_dict['tool'] = TOOL + plugin_dict['id'] = plugin_id + v['plugin_ids'].append(plugin_dict) + + # Set identified by information + identified_dict = dict(models.identified_by_model) + identified_dict['tool'] = TOOL + identified_dict['id'] = plugin_id + v['identified_by'].append(identified_dict) + + # Search for exploits + for exploit in vuln.iter('exploit'): + v['flag'] = True + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = "{0} ({1})".format( + exploit.attrib['type'], + exploit.attrib['id'] + ) + note_dict['content'] = "{0}\n{1}".format( + exploit.attrib['title'], + exploit.attrib['link'] + ) + note_dict['last_modified_by'] = TOOL + v['notes'].append(note_dict) + + # Search for CVE references + for reference in vuln.iter('reference'): + if reference.attrib['source'] == 'CVE': + cve = cve_pattern.sub('', reference.text) + v['cves'].append(cve) + + # Search for solution + solution = vuln.find('solution') + if solution is not None: + for text in solution.itertext(): + s = text.encode('ascii', 'replace').strip() + v['solution'] += white_space_pattern.sub(" ", s) + + # Search for description + description = vuln.find('description') + if description is not None: + for text in description.itertext(): + s = text.encode('ascii', 'replace').strip() + v['description'] += white_space_pattern.sub(" ", s) + + # Build mapping of plugin-id to host to vuln dictionary + vuln_host_map[plugin_id] = dict() + vuln_host_map[plugin_id]['vuln'] = v + vuln_host_map[plugin_id]['hosts'] = set() + + for node in root.iter('node'): + + host_dict = dict(models.host_model) + host_dict['os'] = list() + host_dict['ports'] = list() + host_dict['hostnames'] = list() + + # Set host status + if node.attrib['status'] != 'alive': + host_dict['alive'] = False + + # Set IP address + host_dict['string_addr'] = node.attrib['address'] + host_dict['long_addr'] = helper.ip2long(node.attrib['address']) + + # Set the OS fingerprint + certainty = 0 + for os in node.iter('os'): + if float(os.attrib['certainty']) > certainty: + certainty = float(os.attrib['certainty']) + os_dict = dict(models.os_model) + os_dict['tool'] = TOOL + os_dict['weight'] = OS_WEIGHT + + fingerprint = '' + if 'vendor' in os.attrib: + fingerprint += os.attrib['vendor'] + " " + + # Make an extra check to limit duplication of data in the + # event that the product name was already in the vendor name + if 'product' in os.attrib and \ + os.attrib['product'] not in fingerprint: + fingerprint += os.attrib['product'] + " " + + fingerprint = fingerprint.strip() + os_dict['fingerprint'] = fingerprint + + host_dict['os'] = list() + host_dict['os'].append(os_dict) + + # Test for general, non-port related vulnerabilities + # Add them as tcp, port 0 + tests = node.find('tests') + if tests is not None: + port_dict = dict(models.port_model) + port_dict['service'] = "general" + + for test in tests.findall('test'): + # vulnerable-since attribute is used to flag + # confirmed vulns + if 'vulnerable-since' in test.attrib: + plugin_id = test.attrib['id'].lower() + + # This is used to track evidence for the host/port + # and plugin + h = "{0}:{1}:{2}".format( + host_dict['string_addr'], + "0", + models.PROTOCOL_TCP + ) + vuln_host_map[plugin_id]['hosts'].add(h) + + host_dict['ports'].append(port_dict) + + # Use the endpoint elements to populate port data + for endpoint in node.iter('endpoint'): + port_dict = copy.deepcopy(models.port_model) + port_dict['port'] = int(endpoint.attrib['port']) + port_dict['protocol'] = endpoint.attrib['protocol'] + if endpoint.attrib['status'] != 'open': + port_dict['alive'] = False + + # Use the service elements to identify service + for service in endpoint.iter('service'): + + # Ignore unknown services + if 'unknown' not in service.attrib['name'].lower(): + if not port_dict['service']: + port_dict['service'] = service.attrib['name'].lower() + + # Use the test elements to identify vulnerabilities for + # the host + for test in service.iter('test'): + # vulnerable-since attribute is used to flag + # confirmed vulns + if 'vulnerable-since' in test.attrib: + plugin_id = test.attrib['id'].lower() + + # Add service notes for evidence + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = "{0} (ID{1})".format(plugin_id, + str(note_id)) + for evidence in test.iter(): + if evidence.text: + for line in evidence.text.split("\n"): + line = line.strip() + if line: + note_dict['content'] += " " + \ + line + "\n" + elif evidence.tag == "URLLink": + note_dict['content'] += " " + note_dict['content'] += evidence.attrib[ + 'LinkURL' + ] + "\n" + + note_dict['last_modified_by'] = TOOL + port_dict['notes'].append(note_dict) + note_id += 1 + + # This is used to track evidence for the host/port + # and plugin + h = "{0}:{1}:{2}".format( + host_dict['string_addr'], + str(port_dict['port']), + port_dict['protocol'] + ) + vuln_host_map[plugin_id]['hosts'].add(h) + + # Use the fingerprint elements to identify product + certainty = 0 + for fingerprint in endpoint.iter('fingerprint'): + if float(fingerprint.attrib['certainty']) > certainty: + certainty = float(fingerprint.attrib['certainty']) + prod = '' + if 'vendor' in fingerprint.attrib: + prod += fingerprint.attrib['vendor'] + " " + + if 'product' in fingerprint.attrib: + prod += fingerprint.attrib['product'] + " " + + if 'version' in fingerprint.attrib: + prod += fingerprint.attrib['version'] + " " + + prod = prod.strip() + port_dict['product'] = prod + + host_dict['ports'].append(port_dict) + + project_dict['hosts'].append(host_dict) + + # This code block uses the plugin/host/vuln mapping to associate + # all vulnerable hosts to their vulnerability data within the + # context of the expected Lair schema structure. + for plugin_id, data in vuln_host_map.items(): + + # Build list of host and ports affected by vulnerability and + # assign that list to the vulnerability model + for key in data['hosts']: + (string_addr, port, protocol) = key.split(':') + + host_key_dict = dict(models.host_key_model) + host_key_dict['string_addr'] = string_addr + host_key_dict['port'] = int(port) + host_key_dict['protocol'] = protocol + data['vuln']['hosts'].append(host_key_dict) + + # By default, don't include informational findings unless + # explicitly told to do so. + if data['vuln']['cvss'] == 0 and not include_informational: + continue + + project_dict['vulnerabilities'].append(data['vuln']) + + return project_dict \ No newline at end of file diff --git a/lairdrone/nmap.py b/lairdrone/nmap.py new file mode 100644 index 0000000..5c9245b --- /dev/null +++ b/lairdrone/nmap.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import copy +import xml.etree.ElementTree as et +from lairdrone import drone_models as models +from lairdrone import helper + +OS_WEIGHT = 50 +TOOL = "nmap" + + +def parse(project, resource): + """Parses an Nmap XML file and updates the Lair database + + :param project: The project id + :param resource: The Nmap xml file or xml string to be parsed + """ + + # Attempt to parse resource as file or string + try: + if os.path.isfile(resource): + tree = et.parse(resource) + root = tree.getroot() + else: + root = et.fromstring(resource) + except et.ParseError: + raise + + # Create the project dictionary which acts as foundation of document + project_dict = copy.deepcopy(models.project_model) + project_dict['project_id'] = project + + # Pull the command from the file + command_dict = copy.deepcopy(models.command_model) + command_dict['tool'] = TOOL + + if root.tag == 'nmaprun': + command_dict['command'] = root.attrib['args'] + else: + command = root.find('nmaprun') + if command is not None: + command_dict['command'] = command.attrib['args'] + + project_dict['commands'].append(command_dict) + + # Process each 'host' in the file + for host in root.findall('host'): + + host_dict = copy.deepcopy(models.host_model) + + # Find the host status + status = host.find('status') + if status is not None: + if status.attrib['state'] != 'up': + host_dict['alive'] = False + + # Find the IP address and/or MAC address + for addr in host.findall('address'): + + # Get IP address + if addr.attrib['addrtype'] == 'ipv4': + host_dict['string_addr'] = addr.attrib['addr'] + host_dict['long_addr'] = helper.ip2long(addr.attrib['addr']) + elif addr.attrib['addrtype'] == 'mac': + host_dict['mac_addr'] = addr.attrib['addr'] + + # Find the host names + for hostname in host.iter('hostname'): + host_dict['hostnames'].append(hostname.attrib['name']) + + # Find the ports + for port in host.iter('port'): + port_dict = copy.deepcopy(models.port_model) + port_dict['port'] = int(port.attrib['portid']) + port_dict['protocol'] = port.attrib['protocol'] + + # Find port status + status = port.find('state') + if status is not None: + if status.attrib['state'] != 'open': + continue + port_dict['alive'] = True + + # Find port service and product + service = port.find('service') + if service is not None: + port_dict['service'] = service.attrib['name'] + if 'product' in service.attrib: + port_dict['product'] = service.attrib['product'] + else: + port_dict['product'] = "unknown" + + # Find NSE script output + for script in port.findall('script'): + note_dict = copy.deepcopy(models.note_model) + note_dict['title'] = script.attrib['id'] + note_dict['content'] = script.attrib['output'] + note_dict['last_modified_by'] = TOOL + port_dict['notes'].append(note_dict) + + host_dict['ports'].append(port_dict) + + # Find the Operating System + os_dict = copy.deepcopy(models.os_model) + os_dict['tool'] = TOOL + os_list = list(host.iter('osmatch')) + if os_list: + os_dict['weight'] = OS_WEIGHT + os_dict['fingerprint'] = os_list[0].attrib['name'] + + host_dict['os'].append(os_dict) + + project_dict['hosts'].append(host_dict) + + return project_dict \ No newline at end of file diff --git a/lairdrone/raw.py b/lairdrone/raw.py new file mode 100644 index 0000000..c1ffdb2 --- /dev/null +++ b/lairdrone/raw.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Tom Steele, Dan Kottmann, FishNet Security +# See the file license.txt for copying permission + +import os +import sys +import json +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__), '..')) +) + + +def parse(project, resource): + """Parses a raw JSON file and updates the Lair database + + :param project: The project id + :param resource: The JSON file, string, or dict to be parsed + """ + + # Attempt to parse resource as file or string + if isinstance(resource, str) and os.path.isfile(resource): + with open(resource, "r") as raw_json: + project_dict = json.load(raw_json) + elif isinstance(resource, str): + project_dict = json.loads(resource) + elif isinstance(resource, dict): + project_dict = resource + else: + raise TypeError + + project_dict['project_id'] = project + + return project_dict diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..bc0952e --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from distutils.core import setup + +setup( + name="lairdrone", + version="0.1.7", + author='Dan Kottmann, Tom Steele', + author_email='dan.kottmann@fishnetsecurity.com, thomas.steele@fishnetsecurity.com', + packages=['lairdrone'], + scripts=['bin/drone-nmap', 'bin/drone-nessus', 'bin/drone-nexpose', 'bin/drone-burp', 'bin/drone-raw'], + url='https://github.com/fishnetsecurity/lair', + license='LICENSE.txt', + description='Packages and scripts for use with Lair', + install_requires=[ + "pymongo >= 2.5", + ], + +)