Merge pull request #210 from Patrowl/develop

1.5.6 Clean route and quality review

.sonarcloud.properties

@@ -0,0 +1 @@
+sonar.python.version=3

VERSION

@@ -1 +1 @@
-1.5.4 // PatrowlEngines
+1.5.6 // PatrowlEngines

engines/apivoid/Dockerfile

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.28"
 
 # Create the target repo

engines/arachni/Dockerfile

@@ -1,5 +1,4 @@
 FROM ubuntu:20.04
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Arachni\ \(Patrowl engine\)" Version="1.4.28"
 
 ENV VERSION_FRAMEWORK 1.5.1

engines/burp/engine-burp.py

@@ -205,7 +205,7 @@ def _addto_scanqueue(base_url = None):
     res = {"page": "_addto_scanqueue"}
     url = this.scanurl + '/burp/scanner/scans/active'
     if not (request.args.get('base_url') or base_url):
-        print "_addto_scanqueue(): 'base_url' parameter not set"
+        print("_addto_scanqueue(): 'base_url' parameter not set")
         res.update({"status": "error", "reason": "'base_url' parameter not set"})
         return jsonify(res)
 

engines/censys/engine-censys.py

@@ -1,7 +1,8 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
-import censys.certificates
-import time, OpenSSL, json, os, sys, requests, Queue, threading, ssl, socket, hashlib, urlparse, signal, optparse
+import censys
+import time, OpenSSL, json, os, sys, requests, queue, threading, ssl, socket, hashlib, signal, optparse
+from urllib.parse import urlparse
 from datetime import datetime, timedelta, date
 from flask import Flask, redirect, url_for, jsonify, request, send_from_directory
 
@@ -700,7 +701,7 @@ def _requestor_d(key):
                     if this.scans[action['scan_id']]['totalLeft'] == 0:
                         this.scans[action['scan_id']]['finished_at'] = int(time.time() * 1000)
             except Exception:
-                print sys.exc_info()
+                print(sys.exc_info())
         else:
             time.sleep(1)
 
@@ -718,7 +719,7 @@ def _search_cert(keyword,scan_id, key):
             return False
         except Exception:
             time.sleep(1)
-            print sys.exc_info()
+            print(sys.exc_info())
 
     # for all certificates
     try:
@@ -747,7 +748,7 @@ def _get_view_cert(cert_sha, key):
             return False
         except Exception:
             time.sleep(1)
-            print sys.exc_info()
+            print(sys.exc_info())
     return views
 
 
@@ -888,7 +889,7 @@ def _still_exist(url, serial, port, scan_id):
             break;
         except Exception:
             pass
-            #print sys.exc_info()
+            #print(sys.exc_info())
     return new_serial == int(serial)
 
 
@@ -965,7 +966,7 @@ def _ca_trusted(views,scan_id,keyword,key,chain=[]):
                     return False
                 except Exception:
                     time.sleep(1)
-                    print sys.exc_info()
+                    print(sys.exc_info())
             i = 0
             for ct in cert:
                 if i == 0:
@@ -985,7 +986,7 @@ def _ca_trusted(views,scan_id,keyword,key,chain=[]):
                 return False
             except :
                 time.sleep(1)
-                print sys.exc_info()
+                print(sys.exc_info())
         _ca_trusted(views2,scan_id,keyword,key,chain=chain)
 
     return False
@@ -1014,7 +1015,7 @@ def page_not_found(e):
     return jsonify({"page": "not found"})
 
 def _exit_thread(signum, frame):
-    print "\nClean Thread then exit ..."
+    print("\nClean Thread then exit ...")
     for resq in this.requestor:
         resq._Thread__stop()
     sys.exit(1)

engines/certstream/Dockerfile

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="CertStream\ \(Patrowl engine\)" Version="1.4.27"
 
 # Install dependencies

engines/cortex/Dockerfile

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Cortex\ \(Patrowl engine\)" Version="1.4.27"
 
 # Set the working directory

engines/cybelangel/Dockerfile

@@ -1,6 +1,5 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
-LABEL Name="CybelAngel\ \(Patrowl engine\)" Version="1.4.27"
+LABEL Name="CybelAngel\ \(Patrowl engine\)" Version="1.4.28"
 
 # Create the target repo
 RUN mkdir -p /opt/patrowl-engines/cybelangel

engines/cybelangel/VERSION

@@ -1 +1 @@
-1.4.27
+1.4.28

engines/cybelangel/engine-cybelangel.py

@@ -258,12 +258,12 @@ def start_scan():
 
     scan = {
         'assets': asset_groups,
-        'threads':      [],
-        'options':      data['options'],
-        'scan_id':      scan_id,
-        'status':       "STARTED",
-        'started_at':   int(time.time() * 1000),
-        'findings':     []
+        'threads': [],
+        'options': data['options'],
+        'scan_id': scan_id,
+        'status': "STARTED",
+        'started_at': int(time.time() * 1000),
+        'findings': []
     }
 
     engine.scans.update({scan_id: scan})
@@ -309,8 +309,8 @@ def _scan_malicious_websites(scan_id):
     engine.scans[scan_id]['finished_at'] = int(time.time() * 1000)
     return True
 
-
-def get_report(asset, scan_id):
+@app.route('/engines/cybelangel/getreport/<scan_id>')
+def get_report(scan_id):
     """Get report."""
     result = dict()
     result_file = APP_BASE_DIR / 'results' / f'cybelangel_{scan_id}.json'
@@ -337,16 +337,16 @@ def _parse_results(scan_id):
     for finding in engine.scans[scan_id]["findings"]:
         nb_vulns['medium'] +=  1
         issues.append({
-                "issue_id": len(issues)+1,
-                "severity": "medium",
-                "confidence": "certain",
-                "target": {"addr": [finding['domain']], "protocol": "http", "parent": finding['asset_group']},
-                "title": f"[CybelAngel] New asset found on: {finding['domain']}",
-                "solution": "n/a",
-                "metadata": {"risk": {"criticity": "medium"}},
-                "type": "cybelangel_report",
-                "timestamp": timestamp,
-                "description": f"Domain {finding['domain']} found as a malicious domain name by Cybel Angel",
+            "issue_id": len(issues)+1,
+            "severity": "medium",
+            "confidence": "certain",
+            "target": {"addr": [finding['domain']], "protocol": "http", "parent": finding['asset_group']},
+            "title": f"[CybelAngel] New asset found on: {finding['domain']}",
+            "solution": "n/a",
+            "metadata": {"risk": {"criticity": "medium"}},
+            "type": "cybelangel_report",
+            "timestamp": timestamp,
+            "description": f"Domain {finding['domain']} found as a malicious domain name by Cybel Angel",
         })
 
     summary = {
@@ -403,7 +403,7 @@ def getfindings(scan_id):
     finally:
         # remove the scan from the active scan list
         clean_scan(scan_id)
-        return jsonify(res)
+    return jsonify(res)
 
 
 @app.before_first_request

engines/droopescan/Dockerfile

@@ -1,6 +1,5 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
-LABEL Name="droopescan\ \(Patrowl engine\)" Version="1.4.29"
+LABEL Name="droopescan\ \(Patrowl engine\)" Version="1.4.30"
 
 # Set the working directory
 RUN mkdir -p /opt/patrowl-engines/droopescan

engines/droopescan/VERSION

@@ -1 +1 @@
-1.4.29
+1.4.30

engines/droopescan/engine-droopescan.py

@@ -29,7 +29,7 @@
 APP_PORT = 5021
 APP_MAXSCANS = int(os.environ.get('APP_MAXSCANS', 25))
 APP_ENGINE_NAME = "patrowl-droopescan"
-VERSION = "1.4.27"
+VERSION = "1.4.30"
 
 BASE_DIR = os.path.dirname(os.path.realpath(__file__))
 this = sys.modules[__name__]

engines/eyewitness/Dockerfile

@@ -1,5 +1,4 @@
 FROM phusion/baseimage:bionic-1.0.0
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="EyeWitness\ \(Patrowl engine\)" Version="1.4.28"
 
 

engines/nessus/Dockerfile

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Nessus\ \(Patrowl engine\)" Version="1.4.28"
 
 ENV LOGLEVEL info

engines/nmap/Dockerfile

@@ -1,6 +1,4 @@
 FROM alpine:latest
-
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.34"
 
 # Set the working directory

engines/nmap/Dockerfile.alpine

@@ -1,6 +1,4 @@
 FROM alpine:3.13.5
-
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.0.0"
 
 ARG nmap_ver=7.91

engines/nmap/Dockerfile.ubuntu

@@ -1,6 +1,4 @@
 FROM ubuntu:20.04
-
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.0.0"
 
 # Set the working directory

engines/nmap/VERSION

@@ -1 +1 @@
-1.4.34
+1.4.35

engines/nmap/engine-nmap.py

@@ -220,8 +220,15 @@ def _scan_thread(scan_id):
             cmd += " --min-rate {}".format(options.get(opt_key))
         if opt_key == "max-rtt-timeout":  # /!\ @todo / Security issue: Sanitize parameters here
             cmd += " --max-rtt-timeout {}".format(options.get(opt_key))
+        if opt_key == "max-parallelism":  # /!\ @todo / Security issue: Sanitize parameters here
+            cmd += " --max-parallelism {}".format(options.get(opt_key))
+        if opt_key == "min-hostgroup":  # /!\ @todo / Security issue: Sanitize parameters here
+            cmd += " --min-hostgroup {}".format(options.get(opt_key))
 
     cmd += " -iL " + hosts_filename
+
+    # Optimization trial for online scans
+    # cmd += " -PE --osscan-limit --max-rtt-timeout 100ms --max-parallelism 100 --min-hostgroup 100"
     app.logger.debug('cmd: %s', cmd)
 
     cmd_sec = split(cmd)
@@ -242,6 +249,33 @@ def _scan_thread(scan_id):
     max_timeout = APP_SCAN_TIMEOUT_DEFAULT
     timeout = time.time() + max_timeout
 
+    # while time.time() < timeout:
+    #     if hasattr(proc, 'pid') and psutil.pid_exists(proc.pid) and psutil.Process(proc.pid).status() in ["sleeping", "running"]:
+    #         # Scan is still in progress
+    #         time.sleep(3)
+    #         # print(f'scan {scan_id} still running...')
+    #     else:
+    #         # Scan is finished
+    #         # print(f'scan {scan_id} is finished !')
+
+    #         # Check if the report is available (exists && scan finished)
+    #         report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml"
+    #         if not os.path.exists(report_filename):
+    #             return False
+
+    #         issues, raw_hosts = _parse_report(report_filename, scan_id)
+
+    #         # Check if banner grabbing is requested
+    #         if "banner" in options.keys() and options["banner"] in [True, 1, "true", "1", "y", "yes", "on"]:
+    #             extra_issues = get_service_banner(scan_id, raw_hosts)
+    #             issues.extend(extra_issues)
+
+    #         this.scans[scan_id]["issues"] = deepcopy(issues)
+    #         this.scans[scan_id]["issues_available"] = True
+    #         this.scans[scan_id]["status"] = "FINISHED"
+    #         break
+
+    # return True
     while time.time() < timeout:
         if hasattr(proc, 'pid') and psutil.pid_exists(proc.pid) and psutil.Process(proc.pid).status() in ["sleeping", "running"]:
             # Scan is still in progress
@@ -250,31 +284,37 @@ def _scan_thread(scan_id):
         else:
             # Scan is finished
             # print(f'scan {scan_id} is finished !')
+            break
+
+    # Check if the report is available (exists && scan finished)
+    report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml"
+    if not os.path.exists(report_filename):
+        this.scans[scan_id]["status"] = "FINISHED"  # ERROR ?
+        this.scans[scan_id]["issues_available"] = True
+        return False
 
-            # Check if the report is available (exists && scan finished)
-            report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml"
-            if not os.path.exists(report_filename):
-                return False
-
-            issues, raw_hosts = _parse_report(report_filename, scan_id)
-
-            # Check if banner grabbing is requested
-            if "banner" in options.keys() and options["banner"] in [True, 1, "true", "1", "y", "yes", "on"]:
-                extra_issues = get_service_banner(scan_id, raw_hosts)
-                issues.extend(extra_issues)
+    try:
+        issues, raw_hosts = _parse_report(report_filename, scan_id)
 
-            this.scans[scan_id]["issues"] = deepcopy(issues)
-            this.scans[scan_id]["issues_available"] = True
-            this.scans[scan_id]["status"] = "FINISHED"
+        # Check if banner grabbing is requested
+        if "banner" in options.keys() and options["banner"] in [True, 1, "true", "1", "y", "yes", "on"]:
+            extra_issues = get_service_banner(scan_id, raw_hosts)
+            issues.extend(extra_issues)
 
-            break
+        this.scans[scan_id]["issues"] = deepcopy(issues)
+    except Exception:
+        pass
+    this.scans[scan_id]["issues_available"] = True
+    this.scans[scan_id]["status"] = "FINISHED"
 
     return True
 
 
 @app.route('/engines/nmap/clean')
 def clean():
     res = {"page": "clean"}
+
+    stop()
     this.scans.clear()
     loadconfig()
     res.update({"status": "SUCCESS"})
@@ -290,6 +330,7 @@ def clean_scan(scan_id):
         res.update({"status": "error", "reason": f"scan_id '{scan_id}' not found"})
         return jsonify(res)
 
+    stop_scan(scan_id)
     this.scans.pop(scan_id)
     res.update({"status": "removed"})
     return jsonify(res)
@@ -587,7 +628,6 @@ def _parse_report(filename, scan_id):
         # get ports status - generate issues
         if host.find('ports') is not None:
             for port in host.find('ports'):
-                # for port in host.find('ports'):
                 if port.tag == 'extraports':
                     continue
                 proto = port.get('protocol')

engines/openvas/Dockerfile

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="OpenVAS\ \(Patrowl engine\)" Version="1.4.31"
 
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1

engines/owl_code/Dockerfile

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Patrowl\ Code\ Security\ review\ \(Patrowl engine\)" Version="1.4.28"
 
 # Install dependencies

engines/owl_dns/Dockerfile

@@ -1,7 +1,5 @@
-# FROM ubuntu:18.04
 FROM ubuntu:20.04
-MAINTAINER Patrowl.io "[email protected]"
-LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.4.32"
+LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.4.34"
 
 # Install dependencies
 RUN apt-get update && \

engines/owl_dns/Dockerfile.alpine

@@ -1,5 +1,4 @@
 FROM alpine:latest
-MAINTAINER Patrowl.io "[email protected]"
 LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.0"
 
 # Install dependencies

engines/owl_dns/VERSION

@@ -1 +1 @@
-1.4.32
+1.4.34