From 80bf3d4233a6ad4376a5f65cecf7fd57e98a647c Mon Sep 17 00:00:00 2001 From: Alexander Mironov Date: Fri, 18 Jun 2021 18:09:49 +0400 Subject: [PATCH] Add concurrency --- Cargo.toml | 6 +- README.md | 13 +- src/args.rs | 65 ++++---- src/logic.rs | 432 ++++++++++++++++++++++++++---------------------- src/main.rs | 29 ++-- src/requests.rs | 25 +-- src/structs.rs | 13 +- src/utils.rs | 106 +----------- 8 files changed, 325 insertions(+), 364 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a83c08c..ea94200 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "x8" -version = "1.0.1" +version = "2.0.0" authors = ["Alexander Mironov "] edition = "2018" license = "GPL-3.0-or-later" @@ -13,7 +13,8 @@ readme = "README.md" [dependencies] tokio = { version = "1", features = ["full"] } -reqwest = { version = "0.11", features = ["json", "cookies", "rustls-tls"] } +futures = "0.3.15" +reqwest = { version = "0.11", features = ["json", "cookies", "rustls-tls", "trust-dns"] } regex = "1.3.7" percent-encoding = "2.1.0" lazy_static = "1.4.0" @@ -22,3 +23,4 @@ rand = "0.5.0" colored = "2" diffs = "0.2.1" url = "2.1.1" +parking_lot = "0.11" diff --git a/README.md b/README.md index e552b62..53c4efb 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

Hidden parameters discovery suite written in Rust.

-

+

- [How does it work](#how-does-it-work) - [Features](#features) @@ -33,7 +33,7 @@ Firstly, it makes a few basic requests to learn the target, and then it tries to - Supports 6 main methods: GET, POST, PUT, PATCH, DELETE, HEAD. - Has built in 2 main body types: json, urlencode. - Able to discover parameters with not random value, like admin=true -- Uses fast GNU diff as a response comparer. +- Compares responses line-by-line. - Adds to every request cachebuster by default. # Examples @@ -108,14 +108,14 @@ FLAGS: --disable-colors --disable-custom-parameters Do not check automatically parameters like admin=true --disable-progress-bar - -c, --disable-response-correction Do not beautify responses before processing. Reduces accuracy. + -C, --disable-response-correction Do not beautify responses before processing. Reduces accuracy. --encode Encodes query or body before a request, i.e & -> %26, = -> %3D List of chars to encode: ", `, , <, >, &, #, ;, /, =, % - --external-diff Use external diff instead of internal one -L, --follow-redirects Follow redirections --force Ignore 'binary data detected', 'the page is too huge', 'param_template lacks variables' error messages -h, --help Prints help information + --http2 Use http/2 instead of http/1.1 --insecure Use http instead of https when the request file is used --is-json If the output is valid json and the content type does not contain 'json' keyword - specify this argument for a more accurate search @@ -129,7 +129,7 @@ OPTIONS: -t, --body-type Available: urlencode, json. (default is "urlencode") Can be detected automatically if --body is specified - -l, --diff-location Custom location for external diff. Default: takes from $PATH + -c The number of concurrent requests (default is 1) --custom-parameters Check these parameters with non-random values like true/false yes/no (default is "admin bot captcha debug disable encryption env show sso test waf") @@ -138,7 +138,7 @@ OPTIONS: -d, --delay -H, --header Example: -H 'one:one' 'two:two' - --learn-requests Set the custom number of learning requests. (default is 10) + --learn-requests Set the custom number of learning requests. (default is 9) -m, --max Change the maximum number of parameters. (default is 128/192/256 for query and 512 for body) @@ -155,7 +155,6 @@ OPTIONS: -r, --request The file with raw http request --save-responses Save matched responses to a directory - --tmp-directory Directory for response comparing. Default: /tmp -u, --url You can add a custom injection point with %s --value-size Custom value size. Affects {{random}} variables as well (default is 5) diff --git a/src/args.rs b/src/args.rs index 1b102f7..c425140 100644 --- a/src/args.rs +++ b/src/args.rs @@ -1,10 +1,9 @@ use crate::{structs::Config, utils::{parse_request, adjust_body}}; use clap::{crate_version, App, AppSettings, Arg}; -use std::{collections::HashMap, fs, time::Duration, io::{self, Write}, env::temp_dir}; +use std::{collections::HashMap, fs, time::Duration, io::{self, Write}}; use url::Url; pub fn get_config() -> (Config, usize) { - let tmp_dir_help_message: String = format!("Directory for response comparing. Default: {}", temp_dir().to_str().unwrap_or("/tmp")); let app = App::new("x8") .setting(AppSettings::ArgRequiredElseHelp) @@ -110,7 +109,7 @@ pub fn get_config() -> (Config, usize) { .arg( Arg::with_name("disable-response-correction") .long("disable-response-correction") - .short("c") + .short("C") .help("Do not beautify responses before processing. Reduces accuracy.") ) .arg( @@ -188,29 +187,10 @@ pub fn get_config() -> (Config, usize) { .help("Save matched responses to a directory") .takes_value(true) ) - .arg( - Arg::with_name("tmp-directory") - .long("tmp-directory") - .help(&tmp_dir_help_message) - .takes_value(true) - ) .arg( Arg::with_name("disable-cachebuster") .long("disable-cachebuster") ) - .arg( - Arg::with_name("custom-diff-location") - .long("diff-location") - .short("l") - .help("Custom location for external diff. Default: takes from $PATH") - .requires("external_diff") - .takes_value(true) - ) - .arg( - Arg::with_name("external_diff") - .long("external-diff") - .help("Use external diff instead of internal one") - ) /*.arg( Arg::with_name("verify") .long("verify") @@ -225,7 +205,7 @@ pub fn get_config() -> (Config, usize) { .arg( Arg::with_name("learn_requests_count") .long("learn-requests") - .help("Set the custom number of learning requests. (default is 10)") + .help("Set the custom number of learning requests. (default is 9)") .takes_value(true) ) .arg( @@ -234,6 +214,17 @@ pub fn get_config() -> (Config, usize) { .long("max") .help("Change the maximum number of parameters. (default is 128/192/256 for query and 512 for body)") .takes_value(true) + ) + .arg( + Arg::with_name("concurrency") + .short("c") + .help("The number of concurrent requests (default is 1)") + .takes_value(true) + ) + .arg( + Arg::with_name("http2") + .long("http2") + .help("Use http/2 instead of http/1.1") ); let args = app.clone().get_matches(); @@ -288,7 +279,20 @@ pub fn get_config() -> (Config, usize) { } }, None => { - 10 + 9 + } + }; + + let concurrency: usize = match args.value_of("concurrency") { + Some(val) => match val.parse() { + Ok(val) => val, + Err(_) => { + writeln!(io::stderr(), "Unable to parse 'concurrency' value").ok(); + std::process::exit(1); + } + }, + None => { + 1 } }; @@ -358,12 +362,6 @@ pub fn get_config() -> (Config, usize) { //let origin_cachebuster_value: String = "Origin:https://{{random}}.{{host}}".replace("{{host}}", host); if !args.is_present("disable-cachebuster") { - if !headers.keys().any(|i| i.contains("Accept-Encoding")) { - headers.insert( - String::from("Accept-Encoding"), - String::from("gzip, deflate, {{random}}"), - ); - } if !headers.keys().any(|i| i.contains("Accept")) { headers.insert(String::from("Accept"), String::from("*/*, text/{{random}}")); } @@ -400,7 +398,7 @@ pub fn get_config() -> (Config, usize) { let mut url = args .value_of("url") - .unwrap_or("https://example.com") + .unwrap_or("https://something.something") .to_string(); if !args.is_present("as-body") && url.contains('?') && url.contains('=') && !url.contains("%s") { @@ -490,7 +488,6 @@ pub fn get_config() -> (Config, usize) { replay_once: args.is_present("replay-once"), output_file: args.value_of("output").unwrap_or("").to_string(), save_responses: args.value_of("save-responses").unwrap_or("").to_string(), - tmp_directory: args.value_of("tmp-directory").unwrap_or(temp_dir().to_str().unwrap_or("/tmp")).to_string()+"/", as_body: args.is_present("as-body"), force: args.is_present("force"), disable_response_correction: args.is_present("disable-response-correction"), @@ -504,11 +501,11 @@ pub fn get_config() -> (Config, usize) { disable_cachebuster: args.is_present("disable-cachebuster"), //verify: args.is_present("verify"), delay, - diff_location: args.value_of("custom-diff-location").unwrap_or("diff").to_string(), - external_diff: args.is_present("external_diff"), value_size, learn_requests_count, max, + concurrency, + http2: args.is_present("http2") }; config = if !request.is_empty() { diff --git a/src/logic.rs b/src/logic.rs index fef03cf..bbb2179 100644 --- a/src/logic.rs +++ b/src/logic.rs @@ -1,10 +1,14 @@ use crate::{ requests::{random_request, request}, - structs::{Config, ResponseData, Stable}, + structs::{Config, ResponseData, Stable, FuturesData}, utils::{check_diffs, make_hashmap, random_line, generate_request}, }; use colored::*; +use futures::stream::StreamExt; +use std::sync::Arc; +use parking_lot::Mutex; use reqwest::Client; + use std::{ collections::HashMap, io::{self, Write}, @@ -25,253 +29,291 @@ pub async fn cycles( remaining_params: &mut Vec>, found_params: &mut Vec, ) { - let name1 = random_line(config.value_size); - let name2 = random_line(config.value_size); let all = params.len() / max; + let mut count: usize = 0; + let shared_diffs = Arc::new(Mutex::new(diffs)); + let shared_green_lines = Arc::new(Mutex::new(green_lines)); - for (count, chunk) in params.chunks(max).enumerate() { - let query = &make_hashmap(&chunk, config.value_size); + let futures_data = futures::stream::iter(params.chunks(max).map(|chunk| { + count += 1; + let mut futures_data = FuturesData{ + remaining_params: Vec::new(), + found_params: Vec::new() + }; - let response = request(config, client, query, reflections_count).await; + let found_params: &Vec = &found_params; + let cloned_diffs = Arc::clone(&shared_diffs); + let cloned_green_lines = Arc::clone(&shared_green_lines); - //progress bar - if config.verbose > 0 && !config.disable_progress_bar { - write!( - io::stdout(), - "{} {}/{} \r", - &"-> ".bright_yellow(), - count, - all - ).ok(); + async move { - io::stdout().flush().ok(); - } + let query = &make_hashmap(&chunk, config.value_size); + let response = request(config, client, query, reflections_count).await; - //try to find parameters with different number of reflections - if stable.reflections && first && response.reflected_params.len() < 10 { - for param in response.reflected_params.iter() { - if !found_params.contains(param) { - found_params.push(param.to_string()); - if config.verbose > 0 { - writeln!( - io::stdout(), - "{}: {}", - &"reflects".bright_blue(), - param - ).ok(); - } - } + //progress bar + if config.verbose > 0 && !config.disable_progress_bar { + write!( + io::stdout(), + "{} {}/{} \r", + &"-> ".bright_yellow(), + count, + all + ).ok(); + + io::stdout().flush().ok(); } - } else if stable.reflections && !response.reflected_params.is_empty() { - //check whether there is one not reflected parameter between reflected ones - let mut not_reflected_one: &str = &""; - - if chunk.len() - response.reflected_params.len() == 1 { - for el in chunk.iter() { - if !response.reflected_params.contains(el) { - not_reflected_one = el; - if config.verbose > 1 { + //try to find parameters with different number of reflections + if stable.reflections && first && response.reflected_params.len() < 10 { + for param in response.reflected_params.iter() { + if !found_params.contains(param) { + futures_data.found_params.push(param.to_string()); + if config.verbose > 0 { writeln!( io::stdout(), "{}: {}", - &"not reflected one".bright_cyan(), - ¬_reflected_one - ) - .ok(); + &"reflects".bright_blue(), + param + ).ok(); } } } - } + } else if stable.reflections && !response.reflected_params.is_empty() { + //check whether there is one not reflected parameter between reflected ones + let mut not_reflected_one: &str = &""; - if !not_reflected_one.is_empty() && chunk.len() >= 2 { - found_params.push(not_reflected_one.to_owned()); - } + if chunk.len() - response.reflected_params.len() == 1 { + for el in chunk.iter() { + if !response.reflected_params.contains(el) { + not_reflected_one = el; + if config.verbose > 1 { + writeln!( + io::stdout(), + "{}: {}", + &"not reflected one".bright_cyan(), + ¬_reflected_one + ) + .ok(); + } + } + } + } - if response.reflected_params.len() == 1 { - found_params.push(chunk[0].to_owned()); - } else { - remaining_params.push(chunk.to_vec()); + if !not_reflected_one.is_empty() && chunk.len() >= 2 { + futures_data.found_params.push(not_reflected_one.to_owned()); + } + + if response.reflected_params.len() == 1 { + futures_data.found_params.push(chunk[0].to_owned()); + } else { + futures_data.remaining_params.append(&mut chunk.to_vec()); + } + return futures_data } - continue; - } - if initial_response.code == response.code { - if stable.body { - let new_diffs = check_diffs( - config, - &initial_response.text, - &response.text, - &name1, - &name2, - ); - - for diff in new_diffs { - if !diffs.iter().any(|i| i == &diff) { - if !config.save_responses.is_empty() { - let mut output = generate_request(config, query); - output += &("\n\n--- response ---\n\n".to_owned() + &response.text); - - match std::fs::write( - &(config.save_responses.clone() + "/" + &random_line(10)), - output, - ) { - Ok(_) => (), - Err(err) => { - writeln!( - io::stdout(), - "Unable to write to {}/random_values due to {}", - config.save_responses, - err - ).ok(); - } + if initial_response.code == response.code { + if stable.body { + let new_diffs = check_diffs( + &initial_response.text, + &response.text, + ); + let mut diffs = cloned_diffs.lock(); + + //check whether the new_diff has at least 1 unique diff + //and then check whether it was stored or not + if !new_diffs.iter().all(|i| diffs.contains(i)) { + //the next function with .await will never return if something is locked + //so we need to unlock diffs firstly + drop(diffs); + + let tmp_resp = + random_request(&config, &client, reflections_count, max).await; + + //lock it again + diffs = cloned_diffs.lock(); + + let tmp_diffs = check_diffs( + &initial_response.text, + &tmp_resp.text, + ); + + for diff in tmp_diffs { + if !diffs.iter().any(|i| i == &diff) { + diffs.push(diff); } } + } - if config.verbose > 1 { - writeln!( - io::stdout(), - "{} {} ({})", - response.code, - &response.text.len().to_string().bright_yellow(), - &diff - ).ok(); - } + let mut green_lines = cloned_green_lines.lock(); - match green_lines.get(&diff) { - Some(val) => { - let n_val = *val; - if first || config.verbose == 0 { - green_lines.insert(diff.to_string(), n_val + 1); - } else { - //check whether the diff was stored or not - let tmp_resp = - random_request(&config, &client, reflections_count, max).await; - - let tmp_diffs = check_diffs( - config, - &initial_response.text, - &tmp_resp.text, - &name1, - &name2, - ); - - for diff in tmp_diffs { - if !diffs.iter().any(|i| i == &diff) { - diffs.push(diff); - } - } - } + for diff in new_diffs { + if !diffs.contains(&diff) { + if !config.save_responses.is_empty() { + let mut output = generate_request(config, query); + output += &("\n\n--- response ---\n\n".to_owned() + &response.text); - if n_val > 9 { - diffs.push(diff.to_string()) + match std::fs::write( + &(config.save_responses.clone() + "/" + &random_line(10)), + output, + ) { + Ok(_) => (), + Err(err) => { + writeln!( + io::stdout(), + "Unable to write to {}/random_values due to {}", + config.save_responses, + err + ).ok(); + } } } - _ => { - green_lines.insert(diff.to_string(), 0); - } - } - if chunk.len() == 1 && !found_params.contains(&chunk[0]) { - if config.verbose > 0 { + if config.verbose > 1 { writeln!( io::stdout(), - "{}: page {} -> {} ({})", - chunk[0], - initial_response.text.len(), + "{} {} ({})", + response.code, &response.text.len().to_string().bright_yellow(), &diff ).ok(); } - found_params.push(chunk[0].to_owned()); - break; - } else { - remaining_params.push(chunk.to_vec()); - break; + + //catch some often false-positive diffs within the FIRST cycle + match green_lines.get(&diff) { + Some(val) => { + let n_val = *val; + //if there is one diff through 10 responses - it is a false positive one + if first || config.verbose == 0 { + green_lines.insert(diff.to_string(), n_val + 1); + } else if n_val > 9 { + diffs.push(diff.to_string()) + } + } + _ => { + green_lines.insert(diff.to_string(), 0); + } + } + + if chunk.len() == 1 && !found_params.contains(&chunk[0]) && !futures_data.found_params.contains(&chunk[0]) { + if config.verbose > 0 { + writeln!( + io::stdout(), + "{}: page {} -> {} ({})", + chunk[0], + initial_response.text.len(), + &response.text.len().to_string().bright_yellow(), + &diff + ).ok(); + } + futures_data.found_params.push(chunk[0].to_owned()); + break; + } else { + futures_data.remaining_params.append(&mut chunk.to_vec()); + break; + } } } } - } - } else if chunk.len() == 1 && !found_params.contains(&chunk[0]) { - if config.verbose > 0 { - writeln!( - io::stdout(), - "{}: code {} -> {}", - chunk[0], - initial_response.code, - &response.code.to_string().bright_yellow() - ).ok(); - } - found_params.push(chunk[0].to_owned()); - } else { - if !config.save_responses.is_empty() { - let filename = random_line(10); - let mut output = generate_request(config, query); - output += &("\n\n--- response ---\n\n".to_owned() + &response.text); - - match std::fs::write(&(config.save_responses.clone() + "/" + &filename), output) { - Ok(_) => (), - Err(err) => { + } else if chunk.len() == 1 && !found_params.contains(&chunk[0]) && !futures_data.found_params.contains(&chunk[0]) { + if config.verbose > 0 { + writeln!( + io::stdout(), + "{}: code {} -> {}", + chunk[0], + initial_response.code, + &response.code.to_string().bright_yellow() + ).ok(); + } + futures_data.found_params.push(chunk[0].to_owned()); + } else { + if !config.save_responses.is_empty() { + let filename = random_line(10); + let mut output = generate_request(config, query); + output += &("\n\n--- response ---\n\n".to_owned() + &response.text); + + match std::fs::write(&(config.save_responses.clone() + "/" + &filename), output) { + Ok(_) => (), + Err(err) => { + writeln!( + io::stdout(), + "Unable to write to {}/random_values due to {}", + config.save_responses, + err + ).ok(); + } + } + + if config.verbose > 1 { writeln!( io::stdout(), - "Unable to write to {}/random_values due to {}", - config.save_responses, - err + "{} {} and was saved as {}", + &response.code.to_string().bright_yellow(), + response.text.len(), + &filename ).ok(); } - } - - if config.verbose > 1 { + } else if config.verbose > 1 { writeln!( io::stdout(), - "{} {} and was saved as {}", + "{} {} ", &response.code.to_string().bright_yellow(), - response.text.len(), - &filename + response.text.len() ).ok(); } - } else if config.verbose > 1 { - writeln!( - io::stdout(), - "{} {} ", - &response.code.to_string().bright_yellow(), - response.text.len() - ).ok(); - } - match green_lines.get(&response.code.to_string()) { - Some(val) => { - let n_val = *val; - green_lines.insert(response.code.to_string(), n_val + 1); - if n_val > 50 { - let mut random_params: Vec = Vec::new(); + futures_data.remaining_params.append(&mut chunk.to_vec()); - for _ in 0..max { - random_params.push(random_line(config.value_size)); - } + let mut green_lines = cloned_green_lines.lock(); - let query = make_hashmap( - &random_params[..], - config.value_size, - ); + //to prevent loops when ip got banned or server broke + match green_lines.get(&response.code.to_string()) { + Some(val) => { + let n_val = *val; + green_lines.insert(response.code.to_string(), n_val + 1); + if n_val > 50 { + drop(green_lines); - let check_response = request(config, client, &query, 0).await; + let mut random_params: Vec = Vec::new(); - if check_response.code != initial_response.code { - writeln!( - io::stderr(), - "[!] {} the page became unstable (code)", - &config.url - ).ok(); - std::process::exit(1) + for _ in 0..max { + random_params.push(random_line(config.value_size)); + } + + let query = make_hashmap( + &random_params[..], + config.value_size, + ); + + let check_response = request(config, client, &query, 0).await; + + if check_response.code != initial_response.code { + writeln!( + io::stderr(), + "[!] {} the page became unstable (code)", + &config.url + ).ok(); + std::process::exit(1) + } else { + let mut green_lines = cloned_green_lines.lock(); + green_lines.insert(response.code.to_string(), 0); + } } } - } - _ => { - green_lines.insert(response.code.to_string(), 0); + _ => { + green_lines.insert(response.code.to_string(), 0); + } } } + futures_data + } + })) + .buffer_unordered(config.concurrency) + .collect::>() + .await; - remaining_params.push(chunk.to_vec()); + for instance in futures_data { + for param in instance.found_params { + found_params.push(param) } + remaining_params.push(instance.remaining_params) } } \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index f0259bc..a82d2d2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,9 +16,10 @@ use x8::{ }; #[cfg(windows)] -fn main() { +#[tokio::main] +async fn main() { colored::control::set_virtual_terminal(true).unwrap(); - run() + run().await; } #[cfg(not(windows))] @@ -82,13 +83,14 @@ async fn run() { .danger_accept_invalid_certs(true) .timeout(Duration::from_secs(60)) .http1_title_case_headers() - .use_rustls_tls() .cookie_store(true); + if config.http2 { + client = client.use_rustls_tls(); + } if !config.proxy.is_empty() { client = client.proxy(reqwest::Proxy::all(&config.proxy).unwrap()); } - if !config.follow_redirects { client = client.redirect(reqwest::redirect::Policy::none()); } @@ -99,20 +101,21 @@ async fn run() { .danger_accept_invalid_certs(true) .timeout(Duration::from_secs(60)) .http1_title_case_headers() - .use_rustls_tls() .cookie_store(true); + if config.http2 { + replay_client = replay_client.use_rustls_tls(); + } if !config.replay_proxy.is_empty() { replay_client = replay_client.proxy(reqwest::Proxy::all(&config.replay_proxy).unwrap()); } - if !config.follow_redirects { replay_client = replay_client.redirect(reqwest::redirect::Policy::none()); } let replay_client = replay_client.build().unwrap(); - //generate random query for the first requestca + //generate random query for the first request let query = make_hashmap( &(0..max).map(|_| random_line(config.value_size)).collect::>(), config.value_size, @@ -135,7 +138,12 @@ async fn run() { std::process::exit(1) } - params.append(&mut heuristic(&initial_response.text)); + //params.append(&mut heuristic(&initial_response.text)); + for param in heuristic(&initial_response.text) { + if !params.contains(¶m) { + params.push(param) + } + } if params.len() < max { max = params.len(); @@ -145,7 +153,6 @@ async fn run() { } } - //get reflection count initial_response.reflected_params = Vec::new(); let reflections_count = initial_response @@ -180,7 +187,7 @@ async fn run() { if max == 128 { let response = random_request(&config, &client, reflections_count, max + 64).await; - let (is_code_the_same, new_diffs) = compare(&config, &initial_response, &response); + let (is_code_the_same, new_diffs) = compare(&initial_response, &response); let mut is_the_body_the_same = true; for diff in new_diffs.iter() { @@ -191,7 +198,7 @@ async fn run() { if is_code_the_same && (!stable.body || is_the_body_the_same) { let response = random_request(&config, &client, reflections_count, max + 128).await; - let (is_code_the_same, new_diffs) = compare(&config, &initial_response, &response); + let (is_code_the_same, new_diffs) = compare(&initial_response, &response); for diff in new_diffs { if !diffs.iter().any(|i| i == &diff) { diff --git a/src/requests.rs b/src/requests.rs index 33c60a3..b5031a8 100644 --- a/src/requests.rs +++ b/src/requests.rs @@ -49,7 +49,7 @@ pub async fn empty_reqs( stable.reflections = false; } - let (is_code_the_same, new_diffs) = compare(config, initial_response, &response); + let (is_code_the_same, new_diffs) = compare(initial_response, &response); if !is_code_the_same { writeln!( @@ -69,7 +69,7 @@ pub async fn empty_reqs( let response = random_request(config, client, reflections_count, max).await; - for diff in compare(config, initial_response, &response).1 { + for diff in compare(initial_response, &response).1 { if !diffs.iter().any(|i| i == &diff) { if config.verbose > 0 { writeln!( @@ -185,15 +185,20 @@ pub async fn request( let res = match create_request(url, body.clone(), config, client).send().await { Ok(val) => val, - Err(err) => { - writeln!(io::stderr(), "[!] {} {:?}", url, err).ok(); - writeln!(io::stderr(), "[~] error at the {} observed. Wait 50 sec and repeat.", config.url).ok(); - std::thread::sleep(Duration::from_secs(50)); + Err(_) => { match create_request(url, body.clone(), config, client).send().await { Ok(val) => val, - Err(_) => { - writeln!(io::stderr(), "[!] unable to reach {}", config.url).ok(); - std::process::exit(1); + Err(err) => { + writeln!(io::stderr(), "[!] {} {:?}", url, err).ok(); + writeln!(io::stderr(), "[~] error at the {} observed. Wait 50 sec and repeat.", config.url).ok(); + std::thread::sleep(Duration::from_secs(50)); + match create_request(url, body.clone(), config, client).send().await { + Ok(val) => val, + Err(_) => { + writeln!(io::stderr(), "[!] unable to reach {}", config.url).ok(); + std::process::exit(1); + } + } } } } @@ -204,7 +209,7 @@ pub async fn request( for (key, value) in res.headers().iter() { headers.insert( key.as_str().to_string(), - value.to_str().unwrap().to_string(), + value.to_str().unwrap_or("").to_string(), ); } diff --git a/src/structs.rs b/src/structs.rs index 96f0f90..ca91737 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -21,6 +21,12 @@ pub struct ResponseData { } }*/ +#[derive(Debug, Clone)] +pub struct FuturesData { + pub remaining_params: Vec, + pub found_params: Vec +} + #[derive(Debug, Clone)] pub struct Config { pub method: String, @@ -36,7 +42,6 @@ pub struct Config { pub proxy: String, pub output_file: String, pub save_responses: String, - pub tmp_directory: String, pub force: bool, pub disable_response_correction: bool, pub disable_custom_parameters: bool, @@ -52,11 +57,11 @@ pub struct Config { pub disable_cachebuster: bool, //pub verify: bool, pub delay: Duration, - pub diff_location: String, - pub external_diff: bool, pub value_size: usize, pub learn_requests_count: usize, - pub max: usize + pub max: usize, + pub concurrency: usize, + pub http2: bool } #[derive(Debug)] diff --git a/src/utils.rs b/src/utils.rs index 7699b81..39b1239 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,9 +8,8 @@ use rand::Rng; use regex::Regex; use reqwest::Client; use std::{ -// env, collections::HashMap, - fs::{self, File}, + fs::File, io::{self, BufRead, Write}, path::Path, }; @@ -40,12 +39,9 @@ lazy_static! { //calls check_diffs & returns code and found diffs pub fn compare( - config: &Config, initial_response: &ResponseData, response: &ResponseData, ) -> (bool, Vec) { - let name1 = random_line(3); - let name2 = random_line(3); let mut code: bool = true; let mut diffs: Vec = Vec::new(); @@ -55,11 +51,8 @@ pub fn compare( } for diff in check_diffs( - config, &initial_response.text, &response.text, - &name1, - &name2, ) { diffs.push(diff) } @@ -255,107 +248,18 @@ pub fn make_hashmap( hashmap } -//use external or internal diff to compare responses +//use internal diff to compare responses pub fn check_diffs( - config: &Config, resp1: &str, - resp2: &str, - postfix1: &str, - postfix2: &str, + resp2: &str ) -> Vec { - use std::process::Command; - - // TODO leave only internal diff - if !config.external_diff { - let diffs = match diff(resp1, resp2) { - Ok(val) => val, - Err(err) => { - writeln!(io::stderr(), "Unable to compare: {}", err).ok(); - std::process::exit(1); - } - }; - - return diffs - } - - let mut name1 = config.tmp_directory.clone(); - let mut name2 = config.tmp_directory.clone(); - - name1.push_str(postfix1); - name2.push_str(postfix2); - - let mut diffs: Vec = Vec::new(); - - match fs::write(&name1, resp1) { - Ok(_) => (), - Err(err) => { - writeln!(io::stderr(), "[!] unable to create temp file: {}", err).ok(); - std::process::exit(1); - } - }; - match fs::write(&name2, resp2) { - Ok(_) => (), - Err(err) => { - writeln!(io::stderr(), "[!] unable to create temp file: {}", err).ok(); - std::process::exit(1); - } - }; - - let output = match Command::new(&config.diff_location) - .arg(&name1) - .arg(&name2) - .output() - { + match diff(resp1, resp2) { Ok(val) => val, Err(err) => { - writeln!(io::stderr(), "[!] diff: {}", err).ok(); + writeln!(io::stderr(), "Unable to compare: {}", err).ok(); std::process::exit(1); } - }; - - match fs::remove_file(name1) { - Ok(_) => (), - Err(err) => writeln!(io::stderr(), "[!] unable to remove file: {}", err).unwrap_or(()), - }; - match fs::remove_file(name2) { - Ok(_) => (), - Err(err) => writeln!(io::stderr(), "[!] unable to remove file: {}", err).unwrap_or(()), - }; - - for line in match std::str::from_utf8(&output.stdout) { - Ok(val) => val, - Err(err) => { - writeln!(io::stderr(), "[!] {}", err).ok(); - return Vec::new(); - } - }.split('\n') { - let line = line.to_string(); - - if !line.is_empty() { - match line.chars().next().unwrap() { - '<' => (), - '>' => (), - '\\' => (), - _ => { - if line.len() == 3 { - match &line[..3] { - "---" => (), - _ => diffs.push(line), - } - } else { - diffs.push(line) - } - } - } - } - - if !diffs.is_empty() && diffs[0].contains("Binary files") && !config.force { - writeln!(io::stderr(), "[!] binary data detected").ok(); - std::process::exit(1) - } } - - diffs } pub fn parse_request(insecure: bool, request: &str, config: Config) -> Option {