Merge pull request #170 from HASecuritySolutions/beta-1.8

VulnWhisperer Release 1.8

.github/ISSUE_TEMPLATE/bug_report.md

@@ -11,7 +11,10 @@ assignees: ''
 A clear and concise description of what the bug is.
 
 **Affected module**
-Which one is the module that is not working as expected, e.g. Nessus, Qualys WAS, Qualys VM, OpenVAS, ELK, Jira...) 
+Which one is the module that is not working as expected, e.g. Nessus, Qualys WAS, Qualys VM, OpenVAS, ELK, Jira...). 
+
+**VulnWhisperer debug trail**
+If applicable, paste the VulnWhisperer debug trail of the execution for further detail (execute with '-d' flag).
 
 **To Reproduce**
 Steps to reproduce the behavior:
@@ -27,8 +30,9 @@ A clear and concise description of what you expected to happen.
 If applicable, add screenshots to help explain your problem.
 
 **System in which VulnWhisperer runs (please complete the following information):**
- - OS: [e.g. iOS]
- - Version [e.g. 22]
+ - OS: [e.g. Ubuntu Server]
+ - Version: [e.g. 18.04.2 LTS]
+ - VulnWhisperer Version: [e.g. 1.7.1]
 
 **Additional context**
 Add any other context about the problem here.

.gitignore

@@ -2,7 +2,9 @@
 data/
 logs/
 elk6/vulnwhisperer.ini
+resources/elk6/vulnwhisperer.ini
 configs/frameworks_example.ini
+tests/data
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "qualysapi"]
-	path = deps/qualysapi
-	url = https://github.com/austin-taylor/qualysapi.git
+[submodule "tests/data"]
+	path = tests/data
+	url = https://github.com/HASecuritySolutions/VulnWhisperer-tests.git

.travis.yml

@@ -3,12 +3,22 @@ language: python
 cache: pip
 python:
     - 2.7
+env:
+    - TEST_PATH=tests/data
 
+services:
+  - docker
 #    - 3.6
 #matrix:
 #    allow_failures:
 #        - python: 3.6 - Commenting out testing for Python 3.6 until ready
 
+before_install:
+    - mkdir -p ./data/esdata1
+    - mkdir -p ./data/es_snapshots
+    - sudo chown -R 1000:1000 ./data/es*
+    - docker build -t vulnwhisperer-local .
+    - docker-compose -f docker-compose-test.yml up -d
 install:
     - pip install -r requirements.txt
     - pip install flake8  # pytest  # add another testing frameworks later
@@ -18,7 +28,9 @@ before_script:
     # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
     - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
 script:
-    - true  # pytest --capture=sys  # add other tests here
+    - python setup.py install
+    - bash tests/test-vuln_whisperer.sh
+    - bash tests/test-docker.sh
 notifications:
     on_success: change
     on_failure: change  # `always` will be the setting once code changes slow down

Dockerfile

@@ -2,25 +2,22 @@ FROM centos:latest
 
 MAINTAINER Justin Henderson [email protected]
 
-RUN yum update -y
-RUN yum install -y python python-devel git gcc
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python get-pip.py
+RUN yum update -y && \
+    yum install -y python python-devel git gcc && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py
 
 WORKDIR /opt/VulnWhisperer
 
 COPY requirements.txt requirements.txt
 COPY setup.py setup.py
 COPY vulnwhisp/ vulnwhisp/
 COPY bin/ bin/
-COPY deps/ deps/
 COPY configs/frameworks_example.ini frameworks_example.ini
 
-RUN python setup.py clean --all
-RUN pip install -r requirements.txt
+RUN python setup.py clean --all && \
+    pip install -r requirements.txt
 
-WORKDIR /opt/VulnWhisperer/deps/qualysapi
-RUN python setup.py install
 
 WORKDIR /opt/VulnWhisperer
 RUN python setup.py install

bin/vuln_whisperer

@@ -5,17 +5,20 @@ __author__ = 'Austin Taylor'
 
 from vulnwhisp.vulnwhisp import vulnWhisperer
 from vulnwhisp.base.config import vwConfig
+from vulnwhisp.test.mock import mockAPI
 import os
 import argparse
 import sys
 import logging
 
+
 def isFileValid(parser, arg):
     if not os.path.exists(arg):
         parser.error("The file %s does not exist!" % arg)
     else:
         return arg
 
+
 def main():
 
     parser = argparse.ArgumentParser(description=""" VulnWhisperer is designed to create actionable data from\
@@ -30,46 +33,64 @@ def main():
                         help='JIRA required only! Scan name from scan to report')
     parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
                         help='Prints status out to screen (defaults to True)')
-    parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username')
-    parser.add_argument('-p', '--password', dest='password', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS password')
-    parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output')
-    parser.add_argument('-d', '--debug', action='store_true', help='Enable debugging messages')
+    parser.add_argument('-u', '--username', dest='username', required=False, default=None,
+                        help='The NESSUS username', type=lambda x: x.strip())
+    parser.add_argument('-p', '--password', dest='password', required=False, default=None,
+                        help='The NESSUS password', type=lambda x: x.strip())
+    parser.add_argument('-F', '--fancy', action='store_true',
+                        help='Enable colourful logging output')
+    parser.add_argument('-d', '--debug', action='store_true',
+                        help='Enable debugging messages')
+    parser.add_argument('--mock', action='store_true',
+                        help='Enable mocked API responses')
+    parser.add_argument('--mock_dir', dest='mock_dir', required=False, default=None,
+                        help='Path of test directory')
     args = parser.parse_args()
 
     # First setup logging
     logging.basicConfig(
         stream=sys.stdout,
+        #format only applies when not using -F flag for colouring
+        format='%(levelname)s:%(name)s:%(funcName)s:%(message)s',
         level=logging.DEBUG if args.debug else logging.INFO
     )
-    logger = logging.getLogger(name='main')
+    logger = logging.getLogger()
+    # we set up the logger to log as well to file
+    fh = logging.FileHandler('vulnwhisperer.log')
+    fh.setLevel(logging.DEBUG if args.debug else logging.INFO)
+    fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(fh)
+
     if args.fancy:
         import coloredlogs
         coloredlogs.install(level='DEBUG' if args.debug else 'INFO')
 
+    if args.mock:
+        mock_api = mockAPI(args.mock_dir, args.verbose)
+        mock_api.mock_endpoints()
+
+    exit_code = 0
+
     try:
         if args.config and not args.section:
-
             # this remains a print since we are in the main binary
             print('WARNING: {warning}'.format(warning='No section was specified, vulnwhisperer will scrape enabled modules from config file. \
-						    \nPlease specify a section using -s. \
+                            \nPlease specify a section using -s. \
                                                     \nExample vuln_whisperer -c config.ini -s nessus'))
             logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.')
-	    config = vwConfig(config_in=args.config)
-	    enabled_sections = config.get_sections_with_attribute('enabled')
-
-	    for section in enabled_sections:
-		vw = vulnWhisperer(config=args.config,
-                               profile=section,
-                               verbose=args.verbose,
-                               username=args.username,
-                               password=args.password,
-                               source=args.source,
-                               scanname=args.scanname)
-
-            	vw.whisper_vulnerabilities()
-            # TODO: fix this to NOT be exit 1 unless in error
-	    sys.exit(1)
+
+            config = vwConfig(config_in=args.config)
+            enabled_sections = config.get_sections_with_attribute('enabled')
 
+            for section in enabled_sections:
+                vw = vulnWhisperer(config=args.config,
+                                   profile=section,
+                                   verbose=args.verbose,
+                                   username=args.username,
+                                   password=args.password,
+                                   source=args.source,
+                                   scanname=args.scanname)
+                exit_code += vw.whisper_vulnerabilities()
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,
@@ -79,19 +100,26 @@ def main():
                                password=args.password,
                                source=args.source,
                                scanname=args.scanname)
+            exit_code += vw.whisper_vulnerabilities()
 
-            vw.whisper_vulnerabilities()
-            # TODO: fix this to NOT be exit 1 unless in error
-            sys.exit(1)
+        close_logging_handlers(logger)
+        sys.exit(exit_code)
 
     except Exception as e:
         if args.verbose:
             # this will remain a print since we are in the main binary
             logger.error('{}'.format(str(e)))
             print('ERROR: {error}'.format(error=e))
         # TODO: fix this to NOT be exit 2 unless in error
+        close_logging_handlers(logger)
         sys.exit(2)
 
+    close_logging_handlers(logger)
+
+def close_logging_handlers(logger):
+    for handler in logger.handlers:
+        handler.close()
+        logger.removeFilter(handler)
 
 if __name__ == '__main__':
     main()

configs/frameworks_example.ini

@@ -26,7 +26,7 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys/
+write_path=/opt/VulnWhisperer/data/qualys_web/
 db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
@@ -42,16 +42,10 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys/
+write_path=/opt/VulnWhisperer/data/qualys_vuln/
 db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
-# Set the maximum number of retries each connection should attempt.
-#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
-max_retries = 10
-# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
-template_id = 126024
-
 [detectify]
 #Reference https://developer.detectify.com/
 enabled = false
@@ -74,27 +68,15 @@ write_path=/opt/VulnWhisperer/data/openvas/
 db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
-#[proxy]
-; This section is optional. Leave it out if you're not using a proxy.
-; You can use environmental variables as well: http://www.python-requests.org/en/latest/user/advanced/#proxies
-
-; proxy_protocol set to https, if not specified.
-#proxy_url = proxy.mycorp.com
-
-; proxy_port will override any port specified in proxy_url
-#proxy_port = 8080
-
-; proxy authentication
-#proxy_username = proxyuser
-#proxy_password = proxypass
-
 [jira]
+enabled = false
 hostname = jira-host
 username = username
 password = password
 write_path = /opt/VulnWhisperer/data/jira/
 db_path = /opt/VulnWhisperer/data/database
 verbose = true
+dns_resolv = False
 
 #Sample jira report scan, will automatically be created for existent scans
 #[jira.qualys_vuln.test_scan]

configs/test.ini

@@ -0,0 +1,90 @@
+[nessus]
+enabled=true
+hostname=nessus
+port=443
+username=nessus_username
+password=nessus_password
+write_path=/opt/VulnWhisperer/data/nessus/
+db_path=/opt/VulnWhisperer/data/database
+trash=false
+verbose=true
+
+[tenable]
+enabled=true
+hostname=tenable
+port=443
+username=tenable.io_username
+password=tenable.io_password
+write_path=/opt/VulnWhisperer/data/tenable/
+db_path=/opt/VulnWhisperer/data/database
+trash=false
+verbose=true
+
+[qualys_web]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = false
+hostname = qualys_web
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys_web/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+# Set the maximum number of retries each connection should attempt.
+#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
+max_retries = 10
+# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
+template_id = 126024
+
+[qualys_vuln]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = true
+hostname = qualys_vuln
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys_vuln/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+[detectify]
+#Reference https://developer.detectify.com/
+enabled = false
+hostname = detectify
+#username variable used as apiKey
+username = exampleuser
+#password variable used as secretKey
+password = examplepass
+write_path =/opt/VulnWhisperer/data/detectify/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+
+[openvas]
+enabled = false
+hostname = openvas
+port = 4000
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/openvas/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+[jira]
+enabled = false
+hostname = jira-host
+username = username
+password = password
+write_path = /opt/VulnWhisperer/data/jira/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+dns_resolv = False
+
+#Sample jira report scan, will automatically be created for existent scans
+#[jira.qualys_vuln.test_scan]
+#source = qualys_vuln
+#scan_name = Test Scan
+#jira_project = PROJECT
+; if multiple components, separate by "," = None
+#components = 
+; minimum criticality to report (low, medium, high or critical) = None
+#min_critical_to_report = high
+