Add concurrency

Sh1Yo · Jun 18, 2021 · 80bf3d4 · 80bf3d4
1 parent 5de009c
commit 80bf3d4
Show file tree

Hide file tree

Showing 8 changed files with 325 additions and 364 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "x8"
-version = "1.0.1"
+version = "2.0.0"
 authors = ["Alexander Mironov <[email protected]>"]
 edition = "2018"
 license = "GPL-3.0-or-later"
@@ -13,7 +13,8 @@ readme = "README.md"
 
 [dependencies]
 tokio = { version = "1", features = ["full"] }
-reqwest = { version = "0.11", features = ["json", "cookies", "rustls-tls"] }
+futures = "0.3.15"
+reqwest = { version = "0.11", features = ["json", "cookies", "rustls-tls", "trust-dns"] }
 regex = "1.3.7"
 percent-encoding = "2.1.0"
 lazy_static = "1.4.0"
@@ -22,3 +23,4 @@ rand = "0.5.0"
 colored = "2"
 diffs = "0.2.1"
 url = "2.1.1"
+parking_lot = "0.11"
diff --git a/README.md b/README.md
@@ -10,7 +10,7 @@
 
 <h3 align="center">Hidden parameters discovery suite written in Rust.</h3>
 
-<p align="center"><a href="https://asciinema.org/a/6bLxIIDdBqcgws84clSO35C2y" target="_blank"><img src="https://asciinema.org/a/6bLxIIDdBqcgws84clSO35C2y.svg" /></a></p>
+<p align="center"><a href="https://asciinema.org/a/oAMn0LK0NciNHgzirYJClyB2v" target="_blank"><img src="https://asciinema.org/a/oAMn0LK0NciNHgzirYJClyB2v.svg" /></a></p>
 
 - [How does it work](#how-does-it-work)
 - [Features](#features)
@@ -33,7 +33,7 @@ Firstly, it makes a few basic requests to learn the target, and then it tries to
 - Supports 6 main methods: GET, POST, PUT, PATCH, DELETE, HEAD.
 - Has built in 2 main body types: json, urlencode.
 - Able to discover parameters with not random value, like admin=true
-- Uses fast GNU diff as a response comparer.
+- Compares responses line-by-line.
 - Adds to every request cachebuster by default.
 
 # Examples
@@ -108,14 +108,14 @@ FLAGS:
         --disable-colors
         --disable-custom-parameters      Do not check automatically parameters like admin=true
         --disable-progress-bar
-    -c, --disable-response-correction    Do not beautify responses before processing. Reduces accuracy.
+    -C, --disable-response-correction    Do not beautify responses before processing. Reduces accuracy.
         --encode                         Encodes query or body before a request, i.e & -> %26, = -> %3D
                                          List of chars to encode: ", `, , <, >, &, #, ;, /, =, %
-        --external-diff                  Use external diff instead of internal one
     -L, --follow-redirects               Follow redirections
         --force                          Ignore 'binary data detected', 'the page is too huge', 'param_template lacks
                                          variables' error messages
     -h, --help                           Prints help information
+        --http2                          Use http/2 instead of http/1.1
         --insecure                       Use http instead of https when the request file is used
         --is-json                        If the output is valid json and the content type does not contain 'json'
                                          keyword - specify this argument for a more accurate search
@@ -129,7 +129,7 @@ OPTIONS:
     -t, --body-type <body type>
             Available: urlencode, json. (default is "urlencode")
             Can be detected automatically if --body is specified
-    -l, --diff-location <custom-diff-location>              Custom location for external diff. Default: takes from $PATH
+    -c <concurrency>                                        The number of concurrent requests (default is 1)
         --custom-parameters <custom-parameters>
             Check these parameters with non-random values like true/false yes/no
             (default is "admin bot captcha debug disable encryption env show sso test waf")
@@ -138,7 +138,7 @@ OPTIONS:
 
     -d, --delay <Delay between requests in milliseconds>
     -H, --header <headers>                                  Example: -H 'one:one' 'two:two'
-        --learn-requests <learn_requests_count>             Set the custom number of learning requests. (default is 10)
+        --learn-requests <learn_requests_count>             Set the custom number of learning requests. (default is 9)
     -m, --max <max>
             Change the maximum number of parameters. (default is 128/192/256 for query and 512 for body)
 
@@ -155,7 +155,6 @@ OPTIONS:
 
     -r, --request <request>                                 The file with raw http request
         --save-responses <save-responses>                   Save matched responses to a directory
-        --tmp-directory <tmp-directory>                     Directory for response comparing. Default: /tmp
     -u, --url <url>                                         You can add a custom injection point with %s
         --value-size <value_size>
             Custom value size. Affects {{random}} variables as well (default is 5)

diff --git a/src/args.rs b/src/args.rs
@@ -1,10 +1,9 @@
 use crate::{structs::Config, utils::{parse_request, adjust_body}};
 use clap::{crate_version, App, AppSettings, Arg};
-use std::{collections::HashMap, fs, time::Duration, io::{self, Write}, env::temp_dir};
+use std::{collections::HashMap, fs, time::Duration, io::{self, Write}};
 use url::Url;
 
 pub fn get_config() -> (Config, usize) {
-    let tmp_dir_help_message: String = format!("Directory for response comparing. Default: {}", temp_dir().to_str().unwrap_or("/tmp"));
 
     let app = App::new("x8")
         .setting(AppSettings::ArgRequiredElseHelp)
@@ -110,7 +109,7 @@ pub fn get_config() -> (Config, usize) {
         .arg(
             Arg::with_name("disable-response-correction")
                 .long("disable-response-correction")
-                .short("c")
+                .short("C")
                 .help("Do not beautify responses before processing. Reduces accuracy.")
         )
         .arg(
@@ -188,29 +187,10 @@ pub fn get_config() -> (Config, usize) {
                 .help("Save matched responses to a directory")
                 .takes_value(true)
         )
-        .arg(
-            Arg::with_name("tmp-directory")
-                .long("tmp-directory")
-                .help(&tmp_dir_help_message)
-                .takes_value(true)
-        )
         .arg(
             Arg::with_name("disable-cachebuster")
                 .long("disable-cachebuster")
         )
-        .arg(
-            Arg::with_name("custom-diff-location")
-                .long("diff-location")
-                .short("l")
-                .help("Custom location for external diff. Default: takes from $PATH")
-                .requires("external_diff")
-                .takes_value(true)
-        )
-        .arg(
-            Arg::with_name("external_diff")
-                .long("external-diff")
-                .help("Use external diff instead of internal one")
-        )
         /*.arg(
             Arg::with_name("verify")
                 .long("verify")
@@ -225,7 +205,7 @@ pub fn get_config() -> (Config, usize) {
         .arg(
             Arg::with_name("learn_requests_count")
                 .long("learn-requests")
-                .help("Set the custom number of learning requests. (default is 10)")
+                .help("Set the custom number of learning requests. (default is 9)")
                 .takes_value(true)
         )
         .arg(
@@ -234,6 +214,17 @@ pub fn get_config() -> (Config, usize) {
                 .long("max")
                 .help("Change the maximum number of parameters. (default is 128/192/256 for query and 512 for body)")
                 .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("concurrency")
+                .short("c")
+                .help("The number of concurrent requests (default is 1)")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("http2")
+                .long("http2")
+                .help("Use http/2 instead of http/1.1")
         );
 
     let args = app.clone().get_matches();
@@ -288,7 +279,20 @@ pub fn get_config() -> (Config, usize) {
             }
         },
         None => {
-            10
+            9
+        }
+    };
+
+    let concurrency: usize = match args.value_of("concurrency") {
+        Some(val) => match val.parse() {
+            Ok(val) => val,
+            Err(_) => {
+                writeln!(io::stderr(), "Unable to parse 'concurrency' value").ok();
+                std::process::exit(1);
+            }
+        },
+        None => {
+            1
         }
     };
 
@@ -358,12 +362,6 @@ pub fn get_config() -> (Config, usize) {
     //let origin_cachebuster_value: String = "Origin:https://{{random}}.{{host}}".replace("{{host}}", host);
 
     if !args.is_present("disable-cachebuster") {
-        if !headers.keys().any(|i| i.contains("Accept-Encoding")) {
-            headers.insert(
-                String::from("Accept-Encoding"),
-                String::from("gzip, deflate, {{random}}"),
-            );
-        }
         if !headers.keys().any(|i| i.contains("Accept")) {
             headers.insert(String::from("Accept"), String::from("*/*, text/{{random}}"));
         }
@@ -400,7 +398,7 @@ pub fn get_config() -> (Config, usize) {
 
     let mut url = args
         .value_of("url")
-        .unwrap_or("https://example.com")
+        .unwrap_or("https://something.something")
         .to_string();
 
     if !args.is_present("as-body") && url.contains('?') && url.contains('=') && !url.contains("%s") {
@@ -490,7 +488,6 @@ pub fn get_config() -> (Config, usize) {
         replay_once: args.is_present("replay-once"),
         output_file: args.value_of("output").unwrap_or("").to_string(),
         save_responses: args.value_of("save-responses").unwrap_or("").to_string(),
-        tmp_directory: args.value_of("tmp-directory").unwrap_or(temp_dir().to_str().unwrap_or("/tmp")).to_string()+"/",
         as_body: args.is_present("as-body"),
         force: args.is_present("force"),
         disable_response_correction: args.is_present("disable-response-correction"),
@@ -504,11 +501,11 @@ pub fn get_config() -> (Config, usize) {
         disable_cachebuster: args.is_present("disable-cachebuster"),
         //verify: args.is_present("verify"),
         delay,
-        diff_location: args.value_of("custom-diff-location").unwrap_or("diff").to_string(),
-        external_diff: args.is_present("external_diff"),
         value_size,
         learn_requests_count,
         max,
+        concurrency,
+        http2: args.is_present("http2")
     };
 
     config = if !request.is_empty() {