From 7f2c59f531993ad5d493e6e8ef902444c01ff08e Mon Sep 17 00:00:00 2001 From: qmontal Date: Thu, 5 Jul 2018 19:34:02 +0200 Subject: [PATCH] Qualys Vulnerability Management integration (#74) * Add Qualys vulnerability scans * Use non-zero exit codes for failures * Convert to strings for Logstash * Update logstash config for vulnerability scans * Update README * Grab all scans statuses * Add Qualys vulnerability scans * Use non-zero exit codes for failures * Convert to strings for Logstash * Update logstash config for vulnerability scans * Update README * Grab all scans statuses * Fix error: "Cannot convert non-finite values (NA or inf) to integer" When trying to download the results of Qualys Vulnerability Management scans, the following error pops up: [FAIL] - Could not process scan/xxxxxxxxxx.xxxxx - Cannot convert non-finite values (NA or inf) to integer This error is due to pandas operating with the scan results json file, as the last element from the json doesn't fir with the rest of the response's scheme: that element is "target_distribution_across_scanner_appliances", which contains the scanners used and the IP ranges that each scanner went through. Taking out the last line solves the issue. Also adding the qualys_vuln scheme to the frameworks_example.ini --- README.md | 2 +- configs/frameworks_example.ini | 18 +++- logstash/2000_qualys_web_scans.conf | 22 +++-- vulnwhisp/frameworks/qualys_vuln.py | 114 ++++++++++++++++++++++++ vulnwhisp/vulnwhisp.py | 132 +++++++++++++++++++++++++++- 5 files changed, 276 insertions(+), 12 deletions(-) create mode 100644 vulnwhisp/frameworks/qualys_vuln.py diff --git a/README.md b/README.md index 149a86d..2e32bf6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Currently Supports - [X] [Nessus (v6 & **v7**)](https://www.tenable.com/products/nessus/nessus-professional) - [X] [Qualys Web Applications](https://www.qualys.com/apps/web-app-scanning/) -- [ ] [Qualys Vulnerability Management (Need license)](https://www.qualys.com/apps/vulnerability-management/) +- [X] [Qualys Vulnerability Management (Need license)](https://www.qualys.com/apps/vulnerability-management/) - [X] [OpenVAS](http://www.openvas.org/) - [X] [Tenable.io](https://www.tenable.com/products/tenable-io) - [ ] [Nexpose](https://www.rapid7.com/products/nexpose/) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index a9a6a89..4a74098 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -36,8 +36,24 @@ max_retries = 10 # Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. template_id = 126024 -[openvas] +[qualys_vuln] +#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API enabled = true +hostname = qualysapi.qg2.apps.qualys.com +username = exampleuser +password = examplepass +write_path=/opt/vulnwhisperer/qualys/ +db_path=/opt/vulnwhisperer/database +verbose=true + +# Set the maximum number of retries each connection should attempt. +#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. +max_retries = 10 +# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. +template_id = 126024 + +[openvas] +enabled = false hostname = localhost port = 4000 username = exampleuser diff --git a/logstash/2000_qualys_web_scans.conf b/logstash/2000_qualys_web_scans.conf index cc8b0cb..b3bddb8 100644 --- a/logstash/2000_qualys_web_scans.conf +++ b/logstash/2000_qualys_web_scans.conf @@ -10,12 +10,17 @@ input { type => json codec => json start_position => "beginning" - tags => [ "qualys_web", "qualys" ] + tags => [ "qualys" ] } } filter { - if "qualys_web" in [tags] { + if "qualys" in [tags] { + grok { + match => { "path" => [ "(?qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] } + tag_on_failure => [] + } + mutate { replace => [ "message", "%{message}" ] #gsub => [ @@ -29,15 +34,16 @@ filter { #] } - - grok { - match => { "path" => "qualys_web_%{INT:app_id}_%{INT:last_updated}.json$" } - tag_on_failure => [] - } - + if "qualys_web" in [tags] { mutate { add_field => { "asset" => "%{web_application_name}" } add_field => { "risk_score" => "%{cvss}" } + } + } else if "qualys_vuln" in [tags] { + mutate { + add_field => { "asset" => "%{ip}" } + add_field => { "risk_score" => "%{cvss}" } + } } if [risk] == "1" { diff --git a/vulnwhisp/frameworks/qualys_vuln.py b/vulnwhisp/frameworks/qualys_vuln.py new file mode 100644 index 0000000..01f2104 --- /dev/null +++ b/vulnwhisp/frameworks/qualys_vuln.py @@ -0,0 +1,114 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +__author__ = 'Nathan Young' + +import xml.etree.ElementTree as ET +import pandas as pd +import qualysapi +import requests +import sys +import os +import dateutil.parser as dp + + +class qualysWhisperAPI(object): + SCANS = 'api/2.0/fo/scan' + + def __init__(self, config=None): + self.config = config + try: + self.qgc = qualysapi.connect(config) + # Fail early if we can't make a request or auth is incorrect + self.qgc.request('about.php') + print('[SUCCESS] - Connected to Qualys at %s' % self.qgc.server) + except Exception as e: + print('[ERROR] Could not connect to Qualys - %s' % e) + exit(1) + + def scan_xml_parser(self, xml): + all_records = [] + root = ET.XML(xml) + for child in root.find('.//SCAN_LIST'): + all_records.append({ + 'name': child.find('TITLE').text, + 'id': child.find('REF').text, + 'date': child.find('LAUNCH_DATETIME').text, + 'type': child.find('TYPE').text, + 'duration': child.find('DURATION').text, + 'status': child.find('.//STATE').text, + }) + return pd.DataFrame(all_records) + + def get_all_scans(self): + parameters = { + 'action': 'list', + 'echo_request': 0, + 'show_op': 0, + 'launched_after_datetime': '0001-01-01' + } + scans_xml = self.qgc.request(self.SCANS, parameters) + return self.scan_xml_parser(scans_xml) + + def get_scan_details(self, scan_id=None): + parameters = { + 'action': 'fetch', + 'echo_request': 0, + 'output_format': 'json_extended', + 'mode': 'extended', + 'scan_ref': scan_id + } + scan_json = self.qgc.request(self.SCANS, parameters) + + # First two columns are metadata we already have + # Last column corresponds to "target_distribution_across_scanner_appliances" element + # which doesn't follow the schema and breaks the pandas data manipulation + return pd.read_json(scan_json).iloc[2:-1] + +class qualysUtils: + def __init__(self): + pass + + def iso_to_epoch(self, dt): + return dp.parse(dt).strftime('%s') + + +class qualysVulnScan: + + def __init__( + self, + config=None, + file_in=None, + file_stream=False, + delimiter=',', + quotechar='"', + ): + self.file_in = file_in + self.file_stream = file_stream + self.report = None + self.utils = qualysUtils() + + if config: + try: + self.qw = qualysWhisperAPI(config=config) + except Exception as e: + print('Could not load config! Please check settings for %s' \ + % e) + + if file_stream: + self.open_file = file_in.splitlines() + elif file_in: + self.open_file = open(file_in, 'rb') + + self.downloaded_file = None + + def process_data(self, scan_id=None): + """Downloads a file from Qualys and normalizes it""" + + print('[ACTION] - Downloading scan ID: %s' % scan_id) + scan_report = self.qw.get_scan_details(scan_id=scan_id) + keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base', 'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status', 'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity', 'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference'] + scan_report = scan_report.filter(keep_columns) + scan_report['severity'] = scan_report['severity'].astype(int).astype(str) + scan_report['qid'] = scan_report['qid'].astype(int).astype(str) + + return scan_report diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 94e298b..7ea881a 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -5,6 +5,7 @@ from base.config import vwConfig from frameworks.nessus import NessusAPI from frameworks.qualys import qualysScanReport +from frameworks.qualys_vuln import qualysVulnScan from frameworks.openvas import OpenVAS_API from utils.cli import bcolors import pandas as pd @@ -88,7 +89,7 @@ def __init__( else: self.vprint('{fail} Please specify a database to connect to!'.format(fail=bcolors.FAIL)) - exit(0) + exit(1) self.table_columns = [ 'scan_name', @@ -226,7 +227,7 @@ def __init__( self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, e=e)) - sys.exit(0) + sys.exit(1) @@ -750,6 +751,129 @@ def process_openvas_scans(self): exit(0) +class vulnWhispererQualysVuln(vulnWhispererBase): + + CONFIG_SECTION = 'qualys' + COLUMN_MAPPING = {'cvss_base': 'cvss', + 'cvss3_base': 'cvss3', + 'cve_id': 'cve', + 'os': 'operating_system', + 'qid': 'plugin_id', + 'severity': 'risk', + 'title': 'plugin_name'} + + def __init__( + self, + config=None, + db_name='report_tracker.db', + purge=False, + verbose=None, + debug=False, + username=None, + password=None, + ): + + super(vulnWhispererQualysVuln, self).__init__(config=config) + + self.qualys_scan = qualysVulnScan(config=config) + self.directory_check() + self.scans_to_process = None + + def whisper_reports(self, + report_id=None, + launched_date=None, + scan_name=None, + scan_reference=None, + output_format='json', + cleanup=True): + try: + launched_date + if 'Z' in launched_date: + launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) + report_name = 'qualys_vuln_' + report_id.replace('/','_') \ + + '_{last_updated}'.format(last_updated=launched_date) \ + + '.json' + + relative_path_name = self.path_check(report_name) + + if os.path.isfile(relative_path_name): + #TODO Possibly make this optional to sync directories + file_length = len(open(relative_path_name).readlines()) + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + file_length, + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) + self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO, filename=relative_path_name)) + + else: + print('Processing report ID: %s' % report_id) + vuln_ready = self.qualys_scan.process_data(scan_id=report_id) + vuln_ready['scan_name'] = scan_name + vuln_ready['scan_reference'] = report_id + vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + vuln_ready.shape[0], + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) + + if output_format == 'json': + with open(relative_path_name, 'w') as f: + f.write(vuln_ready.to_json(orient='records', lines=True)) + f.write('\n') + + print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \ + % report_name) + + except Exception as e: + print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e)) + + + def identify_scans_to_process(self): + self.latest_scans = self.qualys_scan.qw.get_all_scans() + if self.uuids: + self.scans_to_process = self.latest_scans.loc[ + (~self.latest_scans['id'].isin(self.uuids)) + & (self.latest_scans['status'] == 'Finished')] + else: + self.scans_to_process = self.latest_scans + self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO, + new=len(self.scans_to_process))) + + + def process_vuln_scans(self): + counter = 0 + self.identify_scans_to_process() + if self.scans_to_process.shape[0]: + for app in self.scans_to_process.iterrows(): + counter += 1 + r = app[1] + print('Processing %s/%s' % (counter, len(self.scans_to_process))) + self.whisper_reports(report_id=r['id'], + launched_date=r['date'], + scan_name=r['name'], + scan_reference=r['type']) + else: + self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO)) + self.conn.close() + exit(0) + class vulnWhisperer(object): @@ -792,3 +916,7 @@ def whisper_vulnerabilities(self): verbose=self.verbose, profile=self.profile) vw.whisper_nessus() + + elif self.profile == 'qualys_vuln': + vw = vulnWhispererQualysVuln(config=self.config) + vw.process_vuln_scans()