diff --git a/README.md b/README.md index 149a86d..2e32bf6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Currently Supports - [X] [Nessus (v6 & **v7**)](https://www.tenable.com/products/nessus/nessus-professional) - [X] [Qualys Web Applications](https://www.qualys.com/apps/web-app-scanning/) -- [ ] [Qualys Vulnerability Management (Need license)](https://www.qualys.com/apps/vulnerability-management/) +- [X] [Qualys Vulnerability Management (Need license)](https://www.qualys.com/apps/vulnerability-management/) - [X] [OpenVAS](http://www.openvas.org/) - [X] [Tenable.io](https://www.tenable.com/products/tenable-io) - [ ] [Nexpose](https://www.rapid7.com/products/nexpose/) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index a9a6a89..4a74098 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -36,8 +36,24 @@ max_retries = 10 # Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. template_id = 126024 -[openvas] +[qualys_vuln] +#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API enabled = true +hostname = qualysapi.qg2.apps.qualys.com +username = exampleuser +password = examplepass +write_path=/opt/vulnwhisperer/qualys/ +db_path=/opt/vulnwhisperer/database +verbose=true + +# Set the maximum number of retries each connection should attempt. +#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. +max_retries = 10 +# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. +template_id = 126024 + +[openvas] +enabled = false hostname = localhost port = 4000 username = exampleuser diff --git a/logstash/2000_qualys_web_scans.conf b/logstash/2000_qualys_web_scans.conf index cc8b0cb..b3bddb8 100644 --- a/logstash/2000_qualys_web_scans.conf +++ b/logstash/2000_qualys_web_scans.conf @@ -10,12 +10,17 @@ input { type => json codec => json start_position => "beginning" - tags => [ "qualys_web", "qualys" ] + tags => [ "qualys" ] } } filter { - if "qualys_web" in [tags] { + if "qualys" in [tags] { + grok { + match => { "path" => [ "(?qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] } + tag_on_failure => [] + } + mutate { replace => [ "message", "%{message}" ] #gsub => [ @@ -29,15 +34,16 @@ filter { #] } - - grok { - match => { "path" => "qualys_web_%{INT:app_id}_%{INT:last_updated}.json$" } - tag_on_failure => [] - } - + if "qualys_web" in [tags] { mutate { add_field => { "asset" => "%{web_application_name}" } add_field => { "risk_score" => "%{cvss}" } + } + } else if "qualys_vuln" in [tags] { + mutate { + add_field => { "asset" => "%{ip}" } + add_field => { "risk_score" => "%{cvss}" } + } } if [risk] == "1" { diff --git a/vulnwhisp/frameworks/qualys_vuln.py b/vulnwhisp/frameworks/qualys_vuln.py new file mode 100644 index 0000000..01f2104 --- /dev/null +++ b/vulnwhisp/frameworks/qualys_vuln.py @@ -0,0 +1,114 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +__author__ = 'Nathan Young' + +import xml.etree.ElementTree as ET +import pandas as pd +import qualysapi +import requests +import sys +import os +import dateutil.parser as dp + + +class qualysWhisperAPI(object): + SCANS = 'api/2.0/fo/scan' + + def __init__(self, config=None): + self.config = config + try: + self.qgc = qualysapi.connect(config) + # Fail early if we can't make a request or auth is incorrect + self.qgc.request('about.php') + print('[SUCCESS] - Connected to Qualys at %s' % self.qgc.server) + except Exception as e: + print('[ERROR] Could not connect to Qualys - %s' % e) + exit(1) + + def scan_xml_parser(self, xml): + all_records = [] + root = ET.XML(xml) + for child in root.find('.//SCAN_LIST'): + all_records.append({ + 'name': child.find('TITLE').text, + 'id': child.find('REF').text, + 'date': child.find('LAUNCH_DATETIME').text, + 'type': child.find('TYPE').text, + 'duration': child.find('DURATION').text, + 'status': child.find('.//STATE').text, + }) + return pd.DataFrame(all_records) + + def get_all_scans(self): + parameters = { + 'action': 'list', + 'echo_request': 0, + 'show_op': 0, + 'launched_after_datetime': '0001-01-01' + } + scans_xml = self.qgc.request(self.SCANS, parameters) + return self.scan_xml_parser(scans_xml) + + def get_scan_details(self, scan_id=None): + parameters = { + 'action': 'fetch', + 'echo_request': 0, + 'output_format': 'json_extended', + 'mode': 'extended', + 'scan_ref': scan_id + } + scan_json = self.qgc.request(self.SCANS, parameters) + + # First two columns are metadata we already have + # Last column corresponds to "target_distribution_across_scanner_appliances" element + # which doesn't follow the schema and breaks the pandas data manipulation + return pd.read_json(scan_json).iloc[2:-1] + +class qualysUtils: + def __init__(self): + pass + + def iso_to_epoch(self, dt): + return dp.parse(dt).strftime('%s') + + +class qualysVulnScan: + + def __init__( + self, + config=None, + file_in=None, + file_stream=False, + delimiter=',', + quotechar='"', + ): + self.file_in = file_in + self.file_stream = file_stream + self.report = None + self.utils = qualysUtils() + + if config: + try: + self.qw = qualysWhisperAPI(config=config) + except Exception as e: + print('Could not load config! Please check settings for %s' \ + % e) + + if file_stream: + self.open_file = file_in.splitlines() + elif file_in: + self.open_file = open(file_in, 'rb') + + self.downloaded_file = None + + def process_data(self, scan_id=None): + """Downloads a file from Qualys and normalizes it""" + + print('[ACTION] - Downloading scan ID: %s' % scan_id) + scan_report = self.qw.get_scan_details(scan_id=scan_id) + keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base', 'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status', 'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity', 'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference'] + scan_report = scan_report.filter(keep_columns) + scan_report['severity'] = scan_report['severity'].astype(int).astype(str) + scan_report['qid'] = scan_report['qid'].astype(int).astype(str) + + return scan_report diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 94e298b..7ea881a 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -5,6 +5,7 @@ from base.config import vwConfig from frameworks.nessus import NessusAPI from frameworks.qualys import qualysScanReport +from frameworks.qualys_vuln import qualysVulnScan from frameworks.openvas import OpenVAS_API from utils.cli import bcolors import pandas as pd @@ -88,7 +89,7 @@ def __init__( else: self.vprint('{fail} Please specify a database to connect to!'.format(fail=bcolors.FAIL)) - exit(0) + exit(1) self.table_columns = [ 'scan_name', @@ -226,7 +227,7 @@ def __init__( self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, e=e)) - sys.exit(0) + sys.exit(1) @@ -750,6 +751,129 @@ def process_openvas_scans(self): exit(0) +class vulnWhispererQualysVuln(vulnWhispererBase): + + CONFIG_SECTION = 'qualys' + COLUMN_MAPPING = {'cvss_base': 'cvss', + 'cvss3_base': 'cvss3', + 'cve_id': 'cve', + 'os': 'operating_system', + 'qid': 'plugin_id', + 'severity': 'risk', + 'title': 'plugin_name'} + + def __init__( + self, + config=None, + db_name='report_tracker.db', + purge=False, + verbose=None, + debug=False, + username=None, + password=None, + ): + + super(vulnWhispererQualysVuln, self).__init__(config=config) + + self.qualys_scan = qualysVulnScan(config=config) + self.directory_check() + self.scans_to_process = None + + def whisper_reports(self, + report_id=None, + launched_date=None, + scan_name=None, + scan_reference=None, + output_format='json', + cleanup=True): + try: + launched_date + if 'Z' in launched_date: + launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) + report_name = 'qualys_vuln_' + report_id.replace('/','_') \ + + '_{last_updated}'.format(last_updated=launched_date) \ + + '.json' + + relative_path_name = self.path_check(report_name) + + if os.path.isfile(relative_path_name): + #TODO Possibly make this optional to sync directories + file_length = len(open(relative_path_name).readlines()) + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + file_length, + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) + self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO, filename=relative_path_name)) + + else: + print('Processing report ID: %s' % report_id) + vuln_ready = self.qualys_scan.process_data(scan_id=report_id) + vuln_ready['scan_name'] = scan_name + vuln_ready['scan_reference'] = report_id + vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + vuln_ready.shape[0], + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) + + if output_format == 'json': + with open(relative_path_name, 'w') as f: + f.write(vuln_ready.to_json(orient='records', lines=True)) + f.write('\n') + + print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \ + % report_name) + + except Exception as e: + print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e)) + + + def identify_scans_to_process(self): + self.latest_scans = self.qualys_scan.qw.get_all_scans() + if self.uuids: + self.scans_to_process = self.latest_scans.loc[ + (~self.latest_scans['id'].isin(self.uuids)) + & (self.latest_scans['status'] == 'Finished')] + else: + self.scans_to_process = self.latest_scans + self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO, + new=len(self.scans_to_process))) + + + def process_vuln_scans(self): + counter = 0 + self.identify_scans_to_process() + if self.scans_to_process.shape[0]: + for app in self.scans_to_process.iterrows(): + counter += 1 + r = app[1] + print('Processing %s/%s' % (counter, len(self.scans_to_process))) + self.whisper_reports(report_id=r['id'], + launched_date=r['date'], + scan_name=r['name'], + scan_reference=r['type']) + else: + self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO)) + self.conn.close() + exit(0) + class vulnWhisperer(object): @@ -792,3 +916,7 @@ def whisper_vulnerabilities(self): verbose=self.verbose, profile=self.profile) vw.whisper_nessus() + + elif self.profile == 'qualys_vuln': + vw = vulnWhispererQualysVuln(config=self.config) + vw.process_vuln_scans()