diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 00000000..50c10dc9 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1 @@ +sonar.python.version=3 diff --git a/VERSION b/VERSION index 993df355..ed932bb4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.4 // PatrowlEngines +1.5.6 // PatrowlEngines diff --git a/engines/apivoid/Dockerfile b/engines/apivoid/Dockerfile index e3acf967..23c5d5c6 100644 --- a/engines/apivoid/Dockerfile +++ b/engines/apivoid/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.28" # Create the target repo diff --git a/engines/arachni/Dockerfile b/engines/arachni/Dockerfile index e40e6189..c138e0e6 100644 --- a/engines/arachni/Dockerfile +++ b/engines/arachni/Dockerfile @@ -1,5 +1,4 @@ FROM ubuntu:20.04 -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Arachni\ \(Patrowl engine\)" Version="1.4.28" ENV VERSION_FRAMEWORK 1.5.1 diff --git a/engines/burp/engine-burp.py b/engines/burp/engine-burp.py index 7873b58b..95ea959e 100644 --- a/engines/burp/engine-burp.py +++ b/engines/burp/engine-burp.py @@ -205,7 +205,7 @@ def _addto_scanqueue(base_url = None): res = {"page": "_addto_scanqueue"} url = this.scanurl + '/burp/scanner/scans/active' if not (request.args.get('base_url') or base_url): - print "_addto_scanqueue(): 'base_url' parameter not set" + print("_addto_scanqueue(): 'base_url' parameter not set") res.update({"status": "error", "reason": "'base_url' parameter not set"}) return jsonify(res) diff --git a/engines/censys/engine-censys.py b/engines/censys/engine-censys.py index b4f09add..93086aab 100644 --- a/engines/censys/engine-censys.py +++ b/engines/censys/engine-censys.py @@ -1,7 +1,8 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- -import censys.certificates -import time, OpenSSL, json, os, sys, requests, Queue, threading, ssl, socket, hashlib, urlparse, signal, optparse +import censys +import time, OpenSSL, json, os, sys, requests, queue, threading, ssl, socket, hashlib, signal, optparse +from urllib.parse import urlparse from datetime import datetime, timedelta, date from flask import Flask, redirect, url_for, jsonify, request, send_from_directory @@ -700,7 +701,7 @@ def _requestor_d(key): if this.scans[action['scan_id']]['totalLeft'] == 0: this.scans[action['scan_id']]['finished_at'] = int(time.time() * 1000) except Exception: - print sys.exc_info() + print(sys.exc_info()) else: time.sleep(1) @@ -718,7 +719,7 @@ def _search_cert(keyword,scan_id, key): return False except Exception: time.sleep(1) - print sys.exc_info() + print(sys.exc_info()) # for all certificates try: @@ -747,7 +748,7 @@ def _get_view_cert(cert_sha, key): return False except Exception: time.sleep(1) - print sys.exc_info() + print(sys.exc_info()) return views @@ -888,7 +889,7 @@ def _still_exist(url, serial, port, scan_id): break; except Exception: pass - #print sys.exc_info() + #print(sys.exc_info()) return new_serial == int(serial) @@ -965,7 +966,7 @@ def _ca_trusted(views,scan_id,keyword,key,chain=[]): return False except Exception: time.sleep(1) - print sys.exc_info() + print(sys.exc_info()) i = 0 for ct in cert: if i == 0: @@ -985,7 +986,7 @@ def _ca_trusted(views,scan_id,keyword,key,chain=[]): return False except : time.sleep(1) - print sys.exc_info() + print(sys.exc_info()) _ca_trusted(views2,scan_id,keyword,key,chain=chain) return False @@ -1014,7 +1015,7 @@ def page_not_found(e): return jsonify({"page": "not found"}) def _exit_thread(signum, frame): - print "\nClean Thread then exit ..." + print("\nClean Thread then exit ...") for resq in this.requestor: resq._Thread__stop() sys.exit(1) diff --git a/engines/certstream/Dockerfile b/engines/certstream/Dockerfile index 08de72b6..17f0fa8e 100644 --- a/engines/certstream/Dockerfile +++ b/engines/certstream/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="CertStream\ \(Patrowl engine\)" Version="1.4.27" # Install dependencies diff --git a/engines/cortex/Dockerfile b/engines/cortex/Dockerfile index ab08228a..78feee9b 100644 --- a/engines/cortex/Dockerfile +++ b/engines/cortex/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Cortex\ \(Patrowl engine\)" Version="1.4.27" # Set the working directory diff --git a/engines/cybelangel/Dockerfile b/engines/cybelangel/Dockerfile index c124ad31..458040d2 100644 --- a/engines/cybelangel/Dockerfile +++ b/engines/cybelangel/Dockerfile @@ -1,6 +1,5 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" -LABEL Name="CybelAngel\ \(Patrowl engine\)" Version="1.4.27" +LABEL Name="CybelAngel\ \(Patrowl engine\)" Version="1.4.28" # Create the target repo RUN mkdir -p /opt/patrowl-engines/cybelangel diff --git a/engines/cybelangel/VERSION b/engines/cybelangel/VERSION index 5e99adfc..c87f986a 100644 --- a/engines/cybelangel/VERSION +++ b/engines/cybelangel/VERSION @@ -1 +1 @@ -1.4.27 +1.4.28 diff --git a/engines/cybelangel/engine-cybelangel.py b/engines/cybelangel/engine-cybelangel.py index 3fffdcfe..946dac02 100755 --- a/engines/cybelangel/engine-cybelangel.py +++ b/engines/cybelangel/engine-cybelangel.py @@ -258,12 +258,12 @@ def start_scan(): scan = { 'assets': asset_groups, - 'threads': [], - 'options': data['options'], - 'scan_id': scan_id, - 'status': "STARTED", - 'started_at': int(time.time() * 1000), - 'findings': [] + 'threads': [], + 'options': data['options'], + 'scan_id': scan_id, + 'status': "STARTED", + 'started_at': int(time.time() * 1000), + 'findings': [] } engine.scans.update({scan_id: scan}) @@ -309,8 +309,8 @@ def _scan_malicious_websites(scan_id): engine.scans[scan_id]['finished_at'] = int(time.time() * 1000) return True - -def get_report(asset, scan_id): +@app.route('/engines/cybelangel/getreport/') +def get_report(scan_id): """Get report.""" result = dict() result_file = APP_BASE_DIR / 'results' / f'cybelangel_{scan_id}.json' @@ -337,16 +337,16 @@ def _parse_results(scan_id): for finding in engine.scans[scan_id]["findings"]: nb_vulns['medium'] += 1 issues.append({ - "issue_id": len(issues)+1, - "severity": "medium", - "confidence": "certain", - "target": {"addr": [finding['domain']], "protocol": "http", "parent": finding['asset_group']}, - "title": f"[CybelAngel] New asset found on: {finding['domain']}", - "solution": "n/a", - "metadata": {"risk": {"criticity": "medium"}}, - "type": "cybelangel_report", - "timestamp": timestamp, - "description": f"Domain {finding['domain']} found as a malicious domain name by Cybel Angel", + "issue_id": len(issues)+1, + "severity": "medium", + "confidence": "certain", + "target": {"addr": [finding['domain']], "protocol": "http", "parent": finding['asset_group']}, + "title": f"[CybelAngel] New asset found on: {finding['domain']}", + "solution": "n/a", + "metadata": {"risk": {"criticity": "medium"}}, + "type": "cybelangel_report", + "timestamp": timestamp, + "description": f"Domain {finding['domain']} found as a malicious domain name by Cybel Angel", }) summary = { @@ -403,7 +403,7 @@ def getfindings(scan_id): finally: # remove the scan from the active scan list clean_scan(scan_id) - return jsonify(res) + return jsonify(res) @app.before_first_request diff --git a/engines/droopescan/Dockerfile b/engines/droopescan/Dockerfile index 2079611e..df4968c6 100644 --- a/engines/droopescan/Dockerfile +++ b/engines/droopescan/Dockerfile @@ -1,6 +1,5 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" -LABEL Name="droopescan\ \(Patrowl engine\)" Version="1.4.29" +LABEL Name="droopescan\ \(Patrowl engine\)" Version="1.4.30" # Set the working directory RUN mkdir -p /opt/patrowl-engines/droopescan diff --git a/engines/droopescan/VERSION b/engines/droopescan/VERSION index 8a40a8cf..5b3274b4 100644 --- a/engines/droopescan/VERSION +++ b/engines/droopescan/VERSION @@ -1 +1 @@ -1.4.29 +1.4.30 diff --git a/engines/droopescan/engine-droopescan.py b/engines/droopescan/engine-droopescan.py index 4668a7e5..2c5b1dff 100644 --- a/engines/droopescan/engine-droopescan.py +++ b/engines/droopescan/engine-droopescan.py @@ -29,7 +29,7 @@ APP_PORT = 5021 APP_MAXSCANS = int(os.environ.get('APP_MAXSCANS', 25)) APP_ENGINE_NAME = "patrowl-droopescan" -VERSION = "1.4.27" +VERSION = "1.4.30" BASE_DIR = os.path.dirname(os.path.realpath(__file__)) this = sys.modules[__name__] diff --git a/engines/eyewitness/Dockerfile b/engines/eyewitness/Dockerfile index 7035b356..bbe1a305 100644 --- a/engines/eyewitness/Dockerfile +++ b/engines/eyewitness/Dockerfile @@ -1,5 +1,4 @@ FROM phusion/baseimage:bionic-1.0.0 -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="EyeWitness\ \(Patrowl engine\)" Version="1.4.28" diff --git a/engines/nessus/Dockerfile b/engines/nessus/Dockerfile index de218ec7..ca224939 100644 --- a/engines/nessus/Dockerfile +++ b/engines/nessus/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Nessus\ \(Patrowl engine\)" Version="1.4.28" ENV LOGLEVEL info diff --git a/engines/nmap/Dockerfile b/engines/nmap/Dockerfile index 4da6ec87..e95bb21a 100644 --- a/engines/nmap/Dockerfile +++ b/engines/nmap/Dockerfile @@ -1,6 +1,4 @@ FROM alpine:latest - -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.34" # Set the working directory diff --git a/engines/nmap/Dockerfile.alpine b/engines/nmap/Dockerfile.alpine index 69ec11c9..4a7e674f 100644 --- a/engines/nmap/Dockerfile.alpine +++ b/engines/nmap/Dockerfile.alpine @@ -1,6 +1,4 @@ FROM alpine:3.13.5 - -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.0.0" ARG nmap_ver=7.91 diff --git a/engines/nmap/Dockerfile.ubuntu b/engines/nmap/Dockerfile.ubuntu index b2f4acd2..49974afe 100644 --- a/engines/nmap/Dockerfile.ubuntu +++ b/engines/nmap/Dockerfile.ubuntu @@ -1,6 +1,4 @@ FROM ubuntu:20.04 - -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.0.0" # Set the working directory diff --git a/engines/nmap/VERSION b/engines/nmap/VERSION index 8b9619b3..47a97679 100644 --- a/engines/nmap/VERSION +++ b/engines/nmap/VERSION @@ -1 +1 @@ -1.4.34 +1.4.35 diff --git a/engines/nmap/engine-nmap.py b/engines/nmap/engine-nmap.py index b9bc6d23..23620d60 100644 --- a/engines/nmap/engine-nmap.py +++ b/engines/nmap/engine-nmap.py @@ -220,8 +220,15 @@ def _scan_thread(scan_id): cmd += " --min-rate {}".format(options.get(opt_key)) if opt_key == "max-rtt-timeout": # /!\ @todo / Security issue: Sanitize parameters here cmd += " --max-rtt-timeout {}".format(options.get(opt_key)) + if opt_key == "max-parallelism": # /!\ @todo / Security issue: Sanitize parameters here + cmd += " --max-parallelism {}".format(options.get(opt_key)) + if opt_key == "min-hostgroup": # /!\ @todo / Security issue: Sanitize parameters here + cmd += " --min-hostgroup {}".format(options.get(opt_key)) cmd += " -iL " + hosts_filename + + # Optimization trial for online scans + # cmd += " -PE --osscan-limit --max-rtt-timeout 100ms --max-parallelism 100 --min-hostgroup 100" app.logger.debug('cmd: %s', cmd) cmd_sec = split(cmd) @@ -242,6 +249,33 @@ def _scan_thread(scan_id): max_timeout = APP_SCAN_TIMEOUT_DEFAULT timeout = time.time() + max_timeout + # while time.time() < timeout: + # if hasattr(proc, 'pid') and psutil.pid_exists(proc.pid) and psutil.Process(proc.pid).status() in ["sleeping", "running"]: + # # Scan is still in progress + # time.sleep(3) + # # print(f'scan {scan_id} still running...') + # else: + # # Scan is finished + # # print(f'scan {scan_id} is finished !') + + # # Check if the report is available (exists && scan finished) + # report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml" + # if not os.path.exists(report_filename): + # return False + + # issues, raw_hosts = _parse_report(report_filename, scan_id) + + # # Check if banner grabbing is requested + # if "banner" in options.keys() and options["banner"] in [True, 1, "true", "1", "y", "yes", "on"]: + # extra_issues = get_service_banner(scan_id, raw_hosts) + # issues.extend(extra_issues) + + # this.scans[scan_id]["issues"] = deepcopy(issues) + # this.scans[scan_id]["issues_available"] = True + # this.scans[scan_id]["status"] = "FINISHED" + # break + + # return True while time.time() < timeout: if hasattr(proc, 'pid') and psutil.pid_exists(proc.pid) and psutil.Process(proc.pid).status() in ["sleeping", "running"]: # Scan is still in progress @@ -250,24 +284,28 @@ def _scan_thread(scan_id): else: # Scan is finished # print(f'scan {scan_id} is finished !') + break + + # Check if the report is available (exists && scan finished) + report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml" + if not os.path.exists(report_filename): + this.scans[scan_id]["status"] = "FINISHED" # ERROR ? + this.scans[scan_id]["issues_available"] = True + return False - # Check if the report is available (exists && scan finished) - report_filename = f"{BASE_DIR}/results/nmap_{scan_id}.xml" - if not os.path.exists(report_filename): - return False - - issues, raw_hosts = _parse_report(report_filename, scan_id) - - # Check if banner grabbing is requested - if "banner" in options.keys() and options["banner"] in [True, 1, "true", "1", "y", "yes", "on"]: - extra_issues = get_service_banner(scan_id, raw_hosts) - issues.extend(extra_issues) + try: + issues, raw_hosts = _parse_report(report_filename, scan_id) - this.scans[scan_id]["issues"] = deepcopy(issues) - this.scans[scan_id]["issues_available"] = True - this.scans[scan_id]["status"] = "FINISHED" + # Check if banner grabbing is requested + if "banner" in options.keys() and options["banner"] in [True, 1, "true", "1", "y", "yes", "on"]: + extra_issues = get_service_banner(scan_id, raw_hosts) + issues.extend(extra_issues) - break + this.scans[scan_id]["issues"] = deepcopy(issues) + except Exception: + pass + this.scans[scan_id]["issues_available"] = True + this.scans[scan_id]["status"] = "FINISHED" return True @@ -275,6 +313,8 @@ def _scan_thread(scan_id): @app.route('/engines/nmap/clean') def clean(): res = {"page": "clean"} + + stop() this.scans.clear() loadconfig() res.update({"status": "SUCCESS"}) @@ -290,6 +330,7 @@ def clean_scan(scan_id): res.update({"status": "error", "reason": f"scan_id '{scan_id}' not found"}) return jsonify(res) + stop_scan(scan_id) this.scans.pop(scan_id) res.update({"status": "removed"}) return jsonify(res) @@ -587,7 +628,6 @@ def _parse_report(filename, scan_id): # get ports status - generate issues if host.find('ports') is not None: for port in host.find('ports'): - # for port in host.find('ports'): if port.tag == 'extraports': continue proto = port.get('protocol') diff --git a/engines/openvas/Dockerfile b/engines/openvas/Dockerfile index 41b5e3e8..ef9a4a8f 100644 --- a/engines/openvas/Dockerfile +++ b/engines/openvas/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="OpenVAS\ \(Patrowl engine\)" Version="1.4.31" ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1 diff --git a/engines/owl_code/Dockerfile b/engines/owl_code/Dockerfile index 52c39c21..d336ca5f 100644 --- a/engines/owl_code/Dockerfile +++ b/engines/owl_code/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Patrowl\ Code\ Security\ review\ \(Patrowl engine\)" Version="1.4.28" # Install dependencies diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index 2d7dfa42..a4e85a4b 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,7 +1,5 @@ -# FROM ubuntu:18.04 FROM ubuntu:20.04 -MAINTAINER Patrowl.io "getsupport@patrowl.io" -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.4.32" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.4.34" # Install dependencies RUN apt-get update && \ diff --git a/engines/owl_dns/Dockerfile.alpine b/engines/owl_dns/Dockerfile.alpine index 40236e1d..1dc1bee0 100644 --- a/engines/owl_dns/Dockerfile.alpine +++ b/engines/owl_dns/Dockerfile.alpine @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.0" # Install dependencies diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index 00bbe72a..8b9619b3 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.4.32 +1.4.34 diff --git a/engines/owl_dns/engine-owl_dns.py b/engines/owl_dns/engine-owl_dns.py index c6c2f454..c8bf6d14 100644 --- a/engines/owl_dns/engine-owl_dns.py +++ b/engines/owl_dns/engine-owl_dns.py @@ -26,7 +26,7 @@ this.resolver = dns.resolver.Resolver() this.resolver.lifetime = this.resolver.timeout = 5.0 -list_nameservers = os.environ.get('NAMESERVER','8.8.8.8,8.8.4.4').split(",") +list_nameservers = os.environ.get('NAMESERVER', '8.8.8.8,8.8.4.4').split(",") this.resolver.nameservers = list_nameservers this.pool = ThreadPoolExecutor(5) @@ -100,7 +100,6 @@ def start_scan(): scan_id = str(data['scan_id']) - # this.scans.update({scan_id: None}) this.scans.update({scan_id: { 'status': 'STARTED', 'started_at': int(time.time() * 1000), @@ -120,7 +119,6 @@ def start_scan(): 'status': "ERROR", }}) this.scans.pop(scan_id, None) - # print(f"Scan job '{scan_id}' out: Scanner not ready") return jsonify(res), 503 # Sanitize args : @@ -141,18 +139,12 @@ def start_scan(): if 'do_whois' in scan['options'].keys() and data['options']['do_whois']: for asset in data["assets"]: if asset["datatype"] in ["domain", "ip"]: - # th = threading.Thread(target=_get_whois, args=(scan_id, asset["value"],)) - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_get_whois, scan_id, asset["value"]) this.scans[scan_id]['futures'].append(th) if 'do_advanced_whois' in scan['options'].keys() and data['options']['do_advanced_whois']: for asset in data["assets"]: if asset["datatype"] == "domain": - # th = threading.Thread(target=_get_whois, args=(scan_id, asset["value"],)) - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_get_whois, scan_id, asset["value"]) this.scans[scan_id]['futures'].append(th) @@ -160,29 +152,18 @@ def start_scan(): if 'do_subdomain_enum' in scan['options'].keys() and data['options']['do_subdomain_enum']: for asset in data["assets"]: if asset["datatype"] == "domain": - # th = threading.Thread(target=_subdomain_enum, args=(scan_id, asset["value"],)) - # th.daemon = True - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_subdomain_enum, scan_id, asset["value"]) this.scans[scan_id]['futures'].append(th) if 'do_subdomains_resolve' in scan['options'].keys() and data['options']['do_subdomains_resolve']: for asset in data["assets"]: if asset["datatype"] == "domain": - # th = threading.Thread(target=_dns_resolve, args=(scan_id, asset["value"], True)) - # th.daemon = True - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_dns_resolve, scan_id, asset["value"], True) this.scans[scan_id]['futures'].append(th) if 'do_dns_resolve' in scan['options'].keys() and data['options']['do_dns_resolve']: for asset in data["assets"]: if asset["datatype"] == "domain": - # th = threading.Thread(target=_dns_resolve, args=(scan_id, asset["value"], False)) - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_dns_resolve, scan_id, asset["value"], False) this.scans[scan_id]['futures'].append(th) @@ -213,19 +194,12 @@ def start_scan(): if 'do_subdomain_bruteforce' in scan['options'].keys() and data['options']['do_subdomain_bruteforce']: for asset in data["assets"]: if asset["datatype"] == "domain": - # th = threading.Thread(target=_subdomain_bruteforce, args=(scan_id, asset["value"],)) - # th.daemon = True - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_subdomain_bruteforce, scan_id, asset["value"]) this.scans[scan_id]['futures'].append(th) if 'do_reverse_dns' in scan['options'].keys() and data['options']['do_reverse_dns']: for asset in data["assets"]: if asset["datatype"] == "ip": - # th = threading.Thread(target=_reverse_dns, args=(scan_id, asset["value"])) - # th.start() - # this.scans[scan_id]['threads'].append(th) th = this.pool.submit(_reverse_dns, scan_id, asset["value"]) this.scans[scan_id]['futures'].append(th) @@ -265,9 +239,6 @@ def start_scan(): "scan_id": scan['scan_id'] } }) - # print(', '.join([a['value'] for a in data['assets']])) - - # print(f"Scan job '{scan_id}' started (threads) !") return jsonify(res) @@ -346,13 +317,11 @@ def _do_dkim_check(scan_id, asset_value): def _perform_spf_check(scan_id,asset_value): dns_records = __dns_resolve_asset(asset_value,"TXT") - #dmarc_records = __dns_resolve_asset("_dmarc."+asset_value,"TXT") - spf_dict = {"no_spf_found":"high", - "spf_lookups": 0 - } - #_do_dmarc_check(spf_dict,dns_records) - #_do_dmarc_check(spf_dict,dmarc_records) - #_do_dkim_check(spf_dict,asset_value) + spf_dict = { + "no_spf_found": "high", + "spf_lookups": 0 + } + for record in dns_records: for value in record["values"]: if "spf" in value: @@ -434,20 +403,23 @@ def _reverse_dns(scan_id, asset): with scan_lock: if 'reverse_dns' not in this.scans[scan_id]['findings'].keys(): this.scans[scan_id]['findings']['reverse_dns'] = {} - this.scans[scan_id]["findings"]["reverse_dns"].update(res) + if bool(res): + this.scans[scan_id]["findings"]["reverse_dns"].update(res) return res def _get_whois(scan_id, asset): res = {} - + is_domain = __is_domain(asset) + is_ip = __is_ip_addr(asset) + # Check the asset is a valid domain name or IP Address - if not __is_domain(asset) and not __is_ip_addr(asset): + if not is_domain and not is_ip: return res - if __is_domain(asset): - w = whois.whois(str(asset)) + if is_domain: + w = whois.whois(asset) if w.domain_name is None: res.update({ asset: {"errors": w} @@ -456,7 +428,7 @@ def _get_whois(scan_id, asset): res.update({ asset: {"raw": {'dict': w, 'text': w.text}, "text": w.text, "type": "domain"} }) - if __is_ip_addr(asset): + if is_ip: w = IPWhois(str(asset).strip()).lookup_rdap() res.update({ asset: {"raw": {'dict': w, 'text': "see raw"}, "text": "see raw", "type": "ip"} @@ -466,7 +438,8 @@ def _get_whois(scan_id, asset): with scan_lock: if 'whois' not in this.scans[scan_id]['findings'].keys(): this.scans[scan_id]['findings']['whois'] = {} - this.scans[scan_id]['findings']['whois'].update(res) + if bool(res): + this.scans[scan_id]['findings']['whois'].update(res) return res @@ -584,13 +557,12 @@ def stop_scan(scan_id): return jsonify(res) for t in this.scans[scan_id]['threads']: - # t._Thread__stop() - # t.terminate() try: t.join() this.scans[scan_id]['threads'].remove(t) except Exception: pass + this.scans[scan_id]['status'] = "STOPPED" this.scans[scan_id]['finished_at'] = int(time.time() * 1000) @@ -612,8 +584,9 @@ def stop(): @app.route('/engines/owl_dns/clean') def clean(): res = {"page": "clean"} + stop() this.scans.clear() - _loadconfig() + # _loadconfig() res.update({"status": "SUCCESS"}) return jsonify(res) @@ -630,25 +603,11 @@ def clean_scan(scan_id): # Terminate thread if any for t in this.scans[scan_id]['threads']: try: - # t._Thread__stop() - # t.terminate() - # print("{}: clean threads '{}'".format(scan_id, t)) t.join() this.scans[scan_id]['threads'].remove(t) except Exception as e: print(e) pass - # - # for t in this.scans[scan_id]['futures']: - # try: - # # t._Thread__stop() - # # t.terminate() - # print("{}: clean futures '{}'".format(scan_id, t)) - # print(dir(t)) - # # t.join() - # except Exception as e: - # print(e) - # pass # Remove Scan for current scans this.scans.pop(scan_id) @@ -668,7 +627,6 @@ def scan_status(scan_id): if 'threads' in this.scans[scan_id]: for t in this.scans[scan_id]['threads']: - # print("scan_status-thread:", t.name, t.native_id) if t.is_alive(): this.scans[scan_id]['status'] = "SCANNING" all_threads_finished = False @@ -730,9 +688,6 @@ def status(): "scanner": this.scanner, "scans": scans}) - # print("thread-count:", threading.active_count()) - # for thread in threading.enumerate(): - # print("{}:{}".format(thread.name, thread.native_id)) return jsonify(res) @@ -964,6 +919,7 @@ def _parse_results(scan_id): create_new_assets = True for subdomain in subdomains_list: + subdomain = subdomain.strip().lower() if any(x in subdomain for x in bad_str) or subdomain.replace(' ', '') == '': continue s = subdomain.replace("From http://PTRarchive.com: ", "") @@ -1209,7 +1165,6 @@ def _parse_results(scan_id): asset, exp_date.date().isoformat(), ", ".join(exp_date.date().isoformat()) - # ", ".join(expiry_dates) ), "raw": scan['findings']['whois'][asset]['raw']['expiration_date'], "solution": "Renew the domain" @@ -1227,7 +1182,6 @@ def _parse_results(scan_id): asset, exp_date.date().isoformat(), ", ".join(exp_date.date().isoformat()) - # ", ".join(expiry_dates) ), "raw": scan['findings']['whois'][asset]['raw']['expiration_date'], "solution": "Renew the domain" @@ -1245,7 +1199,6 @@ def _parse_results(scan_id): asset, exp_date.date().isoformat(), ", ".join(exp_date.date().isoformat()) - # ", ".join(expiry_dates) ), "raw": scan['findings']['whois'][asset]['raw']['expiration_date'], "solution": "Renew the domain" @@ -1263,7 +1216,6 @@ def _parse_results(scan_id): asset, exp_date.date().isoformat(), ", ".join(exp_date.date().isoformat()) - # ", ".join(expiry_dates) ), "raw": scan['findings']['whois'][asset]['raw']['expiration_date'], "solution": "Renew the domain" @@ -1287,6 +1239,8 @@ def _parse_results(scan_id): @app.route('/engines/owl_dns/getfindings/') def getfindings(scan_id): res = {"page": "getfindings", "scan_id": scan_id} + if not scan_id.isdecimal(): + return jsonify({"status": "error", "reason": "scan_id must be numeric digits only"}) # check if the scan_id exists if scan_id not in this.scans.keys(): @@ -1322,6 +1276,8 @@ def getfindings(scan_id): @app.route('/engines/owl_dns/getreport/') def getreport(scan_id): + if not scan_id.isdecimal(): + return jsonify({"status": "error", "reason": "scan_id must be numeric digits only"}) filepath = f"{BASE_DIR}/results/owl_dns_{scan_id}.json" if not os.path.exists(filepath): @@ -1335,7 +1291,6 @@ def _json_serial(obj): JSON serializer for objects not serializable by default json code Used for datetime serialization when the results are written in file """ - if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date): serial = obj.isoformat() return serial diff --git a/engines/owl_dns/modules/dkimsignatures.py b/engines/owl_dns/modules/dkimsignatures.py index 05a4bddd..04cad0b0 100644 --- a/engines/owl_dns/modules/dkimsignatures.py +++ b/engines/owl_dns/modules/dkimsignatures.py @@ -1,4 +1,6 @@ -dkimlist=[ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +dkimlist = [ "s1", "s2", "selector1", @@ -9,5 +11,5 @@ "k1", "mxvault", "dkim", - ] +] diff --git a/engines/owl_dns/modules/dnstwist.py b/engines/owl_dns/modules/dnstwist.py index a322ab45..1137342d 100644 --- a/engines/owl_dns/modules/dnstwist.py +++ b/engines/owl_dns/modules/dnstwist.py @@ -47,9 +47,8 @@ def search_subdomains(scan_id, domain, tld=False, ssdeep=False, geoip=False, mxc try: outs = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True, timeout=timeout) except subprocess.TimeoutExpired: - print("[+] ERROR - Timeout reached ({}s) for cmd: {}".format( - timeout, cmd)) - # print(outs) + print("[+] ERROR - Timeout reached ({}s) for cmd: {}".format(timeout, cmd)) + if json_validator(outs): return domain, json.loads(outs) else: diff --git a/engines/owl_leaks/Dockerfile b/engines/owl_leaks/Dockerfile index 6fbf0d6a..4ec1c83f 100644 --- a/engines/owl_leaks/Dockerfile +++ b/engines/owl_leaks/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Patrowl\ Data\ Leaks\ \(Patrowl engine\)" Version="1.4.28" # Install dependencies diff --git a/engines/owl_request/Dockerfile b/engines/owl_request/Dockerfile index 0ff6a832..8fc107bb 100644 --- a/engines/owl_request/Dockerfile +++ b/engines/owl_request/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="Request\ \(Patrowl engine\)" Version="1.4.26" # Create the target repo diff --git a/engines/pastebin_monitor/Dockerfile b/engines/pastebin_monitor/Dockerfile index e76600ab..ac3194e4 100644 --- a/engines/pastebin_monitor/Dockerfile +++ b/engines/pastebin_monitor/Dockerfile @@ -1,6 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" - LABEL Name="PastebinMonitor\ \(Patrowl engine\)" Version="1.4.27" # Install dependencies diff --git a/engines/shhgit/Dockerfile b/engines/shhgit/Dockerfile index 315eb306..eeca2e63 100644 --- a/engines/shhgit/Dockerfile +++ b/engines/shhgit/Dockerfile @@ -1,6 +1,5 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" -LABEL Name="SHHGit\ \(Patrowl engine\)" Version="1.4.30" +LABEL Name="SHHGit\ \(Patrowl engine\)" Version="1.4.31" # Create the target repo RUN mkdir -p /opt/patrowl-engines/shhgit diff --git a/engines/shhgit/VERSION b/engines/shhgit/VERSION index 5b3274b4..d156665a 100644 --- a/engines/shhgit/VERSION +++ b/engines/shhgit/VERSION @@ -1 +1 @@ -1.4.30 +1.4.31 diff --git a/engines/shhgit/engine-shhgit.py b/engines/shhgit/engine-shhgit.py index 130edd05..8c12f104 100755 --- a/engines/shhgit/engine-shhgit.py +++ b/engines/shhgit/engine-shhgit.py @@ -37,8 +37,7 @@ DATA_BASE_PATH = APP_BASE_DIR / 'data' REPO_BASE_PATH = DATA_BASE_PATH / 'repositories' OUTPUT_BASE_PATH = DATA_BASE_PATH / 'results' - -VERSION = "1.2.0" +VERSION = "1.4.31" logging.basicConfig(level=(logging.DEBUG if APP_DEBUG else logging.INFO)) LOGGER = logging.getLogger('shhgit') @@ -432,7 +431,7 @@ def getfindings(scan_id): finally: # remove the scan from the active scan list clean_scan(scan_id) - return jsonify(res) + return jsonify(res) @app.before_first_request diff --git a/engines/ssllabs/Dockerfile b/engines/ssllabs/Dockerfile index 8dcd472c..b1436834 100644 --- a/engines/ssllabs/Dockerfile +++ b/engines/ssllabs/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="SSL-Labs\ \(Patrowl engine\)" Version="1.4.26" # Install dependencies diff --git a/engines/sslscan/Dockerfile b/engines/sslscan/Dockerfile index 227d3111..b173968a 100644 --- a/engines/sslscan/Dockerfile +++ b/engines/sslscan/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="SSLScan\ \(Patrowl engine\)" Version="1.4.30" ENV CFLAGS "-D__USE_GNU" diff --git a/engines/urlvoid/Dockerfile b/engines/urlvoid/Dockerfile index a0a76cd2..0f26c9d2 100644 --- a/engines/urlvoid/Dockerfile +++ b/engines/urlvoid/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="URLVoid\ \(Patrowl engine\)" Version="1.4.26" # Create the target repo diff --git a/engines/utils/PatrowlEngine.py b/engines/utils/PatrowlEngine.py index 6d4b4cf1..29fc1f54 100644 --- a/engines/utils/PatrowlEngine.py +++ b/engines/utils/PatrowlEngine.py @@ -122,7 +122,6 @@ def clean_scan(self, scan_id): res.update({"scan_id": scan_id}) if scan_id not in self.scans.keys(): - # raise PatrowlEngineExceptions(1002) res.update({"status": "ERROR", "reason": "scan_id '{}' not found".format(scan_id)}) return jsonify(res) @@ -133,7 +132,6 @@ def clean_scan(self, scan_id): def getstatus_scan(self, scan_id): if scan_id not in self.scans.keys(): - raise PatrowlEngineExceptions(1002) return jsonify({ "status": "ERROR", "details": "scan_id '{}' not found".format(scan_id)}) @@ -180,7 +178,6 @@ def stop_scan(self, scan_id): res = {"page": "stop"} if scan_id not in self.scans.keys(): - # raise PatrowlEngineExceptions(1002) res.update({"status": "ERROR", "reason": "scan_id '{}' not found".format(scan_id)}) return jsonify(res) @@ -249,7 +246,6 @@ def init_scan(self, params): def _parse_results(self, scan_id): if scan_id not in self.scans.keys(): - # raise PatrowlEngineExceptions(1002) return jsonify({ "status": "ERROR", "details": "scan_id '{}' not found".format(scan_id)}) @@ -257,7 +253,6 @@ def _parse_results(self, scan_id): issues = [] summary = {} - # scan = self.scans[scan_id] nb_vulns = { "info": 0, "low": 0, @@ -293,7 +288,6 @@ def getfindings(self, scan_id): # check if the scan is finished self.getstatus_scan(scan_id) if scan['status'] != "FINISHED": - # raise PatrowlEngineExceptions(1003) res.update({"status": "ERROR", "reason": "scan_id '{}' not finished (status={})".format(scan_id, scan['status'])}) return jsonify(res) @@ -405,7 +399,7 @@ def add_issue(self, issue): def had_options(self, options): opts = [] - if isinstance(options, str): # is a string + if isinstance(options, str): opts.append(options) elif isinstance(options, list): opts = options diff --git a/engines/utils/PatrowlEngineTest.py b/engines/utils/PatrowlEngineTest.py index 0071e237..6d245937 100644 --- a/engines/utils/PatrowlEngineTest.py +++ b/engines/utils/PatrowlEngineTest.py @@ -125,7 +125,6 @@ def custom_test(self, test_name, assets, scan_policy={}, is_valid=True, max_time try: # check the file name & siza !! assert True == True - #assert r.json()['scan']['status'] == "FINISHED" except AssertionError: print(r.json()) assert False diff --git a/engines/virustotal/Dockerfile b/engines/virustotal/Dockerfile index b30474f0..c21ee8ca 100644 --- a/engines/virustotal/Dockerfile +++ b/engines/virustotal/Dockerfile @@ -1,5 +1,4 @@ FROM alpine:latest -MAINTAINER Patrowl.io "getsupport@patrowl.io" LABEL Name="VirusTotal\ \(Patrowl engine\)" Version="1.4.28" # Create the target repo diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 00000000..581d09a8 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,2 @@ +sonar.qualitygate.wait=true +sonar.python.version=3 \ No newline at end of file