Stuff, stuff, stuff.

Cargo.toml

@@ -21,7 +21,7 @@ atty = { version = "0.2.14", optional = true }
 thiserror = "1.0.58"
 regex = { version = "1.10.4", optional = true }
 glob = { version = "0.3.1", optional = true }
-psl = "2.1.29"
+psl = "2.1.30"
 form_urlencoded = "1.2.1"
 regex-syntax = { version = "0.8.3", optional = true }
 percent-encoding = { version = "2.3.1", optional = true }
@@ -37,7 +37,7 @@ crate-type = ["cdylib", "rlib"]
 
 [features]
 default = ["http", "commands", "cache-redirects", "stdin", "default-config", "minify-included-strings",
-  "regex", "glob", "string-location", "string-matcher", "string-modification", "string-source", "advanced-requests"]
+  "regex", "glob", "string-location", "string-matcher", "string-modification", "string-source", "advanced-requests", "socks-proxy"]
 # Enables HTTP stuff. DOES NOT WORK ON WASM.
 http = ["dep:reqwest"]
 # Enables [`url_cleaner::glue::CommandWrapper`].
@@ -67,6 +67,8 @@ string-modification = ["dep:percent-encoding"]
 string-source = []
 # Enables [`types::RequestConfig`].
 advanced-requests = ["http", "reqwest/json", "reqwest/cookies"]
+# Enables using SOCKS proxies in [`HttpClientConfig`].
+socks-proxy = ["http", "reqwest/socks"]
 
 # https://stackoverflow.com/a/54842093/10720231
 [profile.release]

README.md

@@ -69,7 +69,6 @@ If the URL in your web browser looks like `file:///run/...` and the webpage is w
 - [`Option<...>`](https://doc.rust-lang.org/std/option/enum.Option.html) just means a value can be `null` in the JSON. `{"abc": "xyz"}` and `{"abc": null}` are both valid states for a `abc: Option<String>` field.
 - [`Box<...>`](https://doc.rust-lang.org/std/boxed/struct.Box.html) has no bearing on JSON syntax or possible values. It's just used so Rust can put types inside themselves.
 - [`Vec<...>`](https://doc.rust-lang.org/std/vec/struct.Vec.html) and [`HashSet<...>`](https://doc.rust-lang.org/std/collections/struct.HashSet.html) are written as lists.
-- [`[...; N]`](https://doc.rust-lang.org/std/primitive.array.html) is written as a list of length `N`.
 - [`HashMap<..., ...>`](https://doc.rust-lang.org/std/collections/struct.HashMap.html) and [`HeaderMap`](https://docs.rs/reqwest/latest/reqwest/header/struct.HeaderMap.html) are written as maps in JSON.
     - [`HeaderMap`](https://docs.rs/reqwest/latest/reqwest/header/struct.HeaderMap.html) keys are always lowercase.
 - Fields preceded by [`#[serde(default)]`](https://serde.rs/field-attrs.html#default) or [`#[serde(default = "...")]`](https://serde.rs/field-attrs.html#default--path) can be omitted from config files. The defaults are almost always what you want.

default-config.json

@@ -564,8 +564,8 @@
 			"condition": {"All": [
 				{"QualifiedDomain": "bsky.app"},
 				{"PartIs": {"part": {"PathSegment": 0}, "value": "profile"}},
-				{"Not": {"PartContains": {"part": {"PathSegment": 1}, "where": "Anywhere", "value": "."}}},
-				{"Not": {"PartContains": {"part": {"PathSegment": 1}, "where": "Anywhere", "value": ":"}}}
+				{"Not": {"PartContains": {"part": {"PathSegment": 1}, "value": "."}}},
+				{"Not": {"PartContains": {"part": {"PathSegment": 1}, "value": ":"}}}
 			]},
 			"mapper": {"ModifyPart": {"part": {"PathSegment": 1}, "how": {"Append": ".bsky.social"}}}
 		},

src/glue.rs

@@ -6,6 +6,10 @@
 #[cfg(feature = "glob"    )] pub use glob::*;
 #[cfg(feature = "commands")] mod command;
 #[cfg(feature = "commands")] pub use command::*;
+#[cfg(all(feature = "http", not(target_family = "wasm")))] pub mod proxy;
+#[cfg(all(feature = "http", not(target_family = "wasm")))] pub use proxy::*;
 /// Serializing and deserializing [`reqwest::header::HeaderMap`].
 #[cfg(all(feature = "http", not(target_family = "wasm")))]
 pub(crate) mod headermap;
+#[cfg(all(feature = "http", not(target_family = "wasm")))]
+pub(crate) mod headervalue;

src/glue/command.rs

@@ -99,7 +99,7 @@ impl CommandConfig {
     pub fn make_command(&self, url: Option<&Url>) -> Command {
         let mut ret = Command::new(&self.program);
         match url {
-            // I don't think [`OsStr::from_bytes`] even helps here, but it maoves the problem into [`Command::args`]'s implementation details and it makes me feel better.
+            // I don't think [`OsStr::from_bytes`] even helps here, but it moves the problem into [`Command::args`]'s implementation details and it makes me feel better.
             // It's the electric car model of programming.
             #[cfg(target_family = "unix")]
             Some(url) => {ret.args(self.args.iter().map(|arg| if &**arg=="{}" {OsStr::from_bytes(url.as_str().as_bytes())} else {OsStr::from_bytes(arg.as_bytes())}));},
@@ -135,6 +135,7 @@ impl CommandConfig {
     /// Run the command from [`Self::make_command`], handle its output using [`Self::output_handler`], and returns the output.
     /// # Errors
     /// If `stdin` is `Some` and the calls to [`Command::spawn`], [`std::process::ChildStdin::write_all`], or [`std::process::Child::wait_with_output`] returns an error, that error is returned.
+    /// 
     /// If `stdin` is `None` and the call to [`Command::output`] returns an error, that error is returned.
     #[allow(clippy::missing_panics_doc)]
     pub fn output(&self, url: Option<&Url>, stdin: Option<&[u8]>) -> Result<String, CommandError> {
@@ -160,6 +161,7 @@ impl CommandConfig {
     /// Runs the command, does the [`OutputHandler`] stuff, trims trailing newlines and carriage returns form the output using [`str::trim_end_matches`], then extracts the URL.
     /// # Errors
     /// If the call to [`Self::output`] returns an error, that error is returned.
+    /// 
     /// If the trimmed output cannot be parsed as a URL, returns the error [`CommandError::UrlParseError`].
     pub fn get_url(&self, url: Option<&Url>) -> Result<Url, CommandError> {
         Ok(Url::parse(self.output(url, None)?.trim_end_matches(&['\r', '\n']))?)

src/glue/glob.rs

@@ -1,4 +1,4 @@
-//! Provides [`GlobWrapper`], a serializaable/deserializable wrapper around [`Pattern`] and [`MatchOptions`].
+//! Provides [`GlobWrapper`], a serializable/deserializable wrapper around [`Pattern`] and [`MatchOptions`].
 
 use std::str::FromStr;
 
@@ -70,7 +70,7 @@ fn deserialize_pattern<'de, D: Deserializer<'de>>(deserializer: D) -> Result<Pat
     Pattern::new(&pattern).map_err(D::Error::custom)
 }
 
-/// Serializaer to turn a [`Pattern`] into a string.
+/// Serializer to turn a [`Pattern`] into a string.
 fn serialize_pattern<S: Serializer>(pattern: &Pattern, serializer: S) -> Result<S::Ok, S::Error> {
     serializer.serialize_str(pattern.as_str())
 }

src/glue/headervalue.rs

@@ -0,0 +1,20 @@
+//! Provides serialization and deserialization functions for [`HeaderValue`].
+
+use serde::{Deserialize, ser::{Serializer, Error as _}, de::{Deserializer, Error as _}};
+#[allow(unused_imports)] // [`HeaderValue`] is imported for [`serialize`]'s documentation.
+use reqwest::header::HeaderValue;
+
+/// Deserializes a [`HeaderValue`]
+/// # Errors
+/// If one of the keys or values aren't a valid header key or value, this function errors.
+pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<HeaderValue, D::Error> {
+    let temp: String = Deserialize::deserialize(d)?;
+    temp.try_into().map_err(D::Error::custom)
+}
+
+/// Serializes [`HeaderValue`].
+/// # Errors
+/// When the call to [`HeaderValue::to_str`] returns an error, that error is returned.
+pub fn serialize<S: Serializer>(x: &HeaderValue, s: S) -> Result<S::Ok, S::Error> {
+    s.serialize_str(x.to_str().map_err(S::Error::custom)?)
+}

src/glue/proxy.rs

@@ -0,0 +1,67 @@
+//! Proxy support for HTTP and HTTPS requests.
+
+use serde::{Serialize, Deserialize};
+use url::Url;
+use reqwest::header::HeaderValue;
+use reqwest::Proxy;
+
+// Used for doc links.
+#[allow(unused_imports)]
+use crate::types::HttpClientConfig;
+
+/// Used by [`HttpClientConfig`] to detail how a [`reqwest::Proxy`] should be made.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct ProxyConfig {
+    /// The URL to proxy traffic to. Not the URL whose traffic to proxy.
+    pub url: Url,
+    /// The type of requests to proxy. Defaults to [`ProxyMode::All`] which proxies HTTP and HTTPS requests.
+    #[serde(default)]
+    pub mode: ProxyMode,
+    /// Authentication for the proxy server. Defaults to [`None`].
+    #[serde(default)]
+    pub auth: Option<ProxyAuth>
+}
+
+/// The types of traffic to proxy. Defaults to [`Self::All`].
+#[derive(Debug, Clone, Copy, PartialEq, Default, Eq, Serialize, Deserialize)]
+pub enum ProxyMode {
+    /// [`reqwest::Proxy::all`].
+    #[default]
+    All,
+    /// [`reqwest::Proxy::https`].
+    Https,
+    /// [`reqwest::Proxy::http`].
+    Http
+}
+
+/// Authentication for the proxy server.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ProxyAuth {
+    /// [`reqwest::Proxy::basic_auth`].
+    Basic {
+        /// The username to use.
+        username: String,
+        /// The password to use.
+        password: String
+    },
+    /// [`reqwest::Proxy::custom_http_auth`].
+    Custom(#[serde(with = "crate::glue::headervalue")] HeaderValue)
+}
+
+impl ProxyConfig {
+    /// Create a [`reqwest::Proxy`].
+    /// # Errors
+    /// If the call to [`reqwest::Proxy::all`], [`reqwest::Proxy::https`], or [`reqwest::Proxy::http`] return an error, that error is returned.
+    pub fn make(&self) -> reqwest::Result<reqwest::Proxy> {
+        let temp = match self.mode {
+            ProxyMode::All   => Proxy::all  (self.url.clone()),
+            ProxyMode::Https => Proxy::https(self.url.clone()),
+            ProxyMode::Http  => Proxy::http (self.url.clone())
+        }?;
+        Ok(match &self.auth {
+            None => temp,
+            Some(ProxyAuth::Basic {username, password}) => temp.basic_auth(username, password),
+            Some(ProxyAuth::Custom(value)) => temp.custom_http_auth(value.clone())
+        })
+    }
+}

src/glue/regex/regex_parts.rs

@@ -81,6 +81,7 @@ impl RegexParts {
     /// Creates a [`RegexParts`] with the provided pattern. The config is set to its default value.
     /// # Errors
     /// If the pattern is invalid, the error encountered by the parser is returned.
+    /// 
     /// The error is boxed because it's massive.
     pub fn new(pattern: &str) -> Result<Self, Box<RegexSyntaxError>> {
         Self::new_with_config(pattern, RegexConfig::default())
@@ -99,6 +100,7 @@ impl RegexParts {
     /// Creates a [`RegexParts`] with the provided pattern and config.
     /// # Errors
     /// If the pattern is invalid, the error encountered by the parser is returned.
+    /// 
     /// The error is boxed because it's massive.
     pub fn new_with_config(pattern: &str, config: RegexConfig) -> Result<Self, Box<RegexSyntaxError>> {
         config.build_parser().parse(pattern).map_err(Box::new)?;

src/main.rs

@@ -16,7 +16,7 @@ mod util;
 #[command(name = "URL Cleaner")]
 /// URL Cleaner is a tool meant primarily to remove tracking garbage from URLs and more generally is a very powerful URL manipulation library/CLI tool.
 struct Args {
-    /// The URLs to cleam before the URLs in the STDIN.
+    /// The URLs to clean before the URLs in the STDIN.
     urls: Vec<Url>,
     /// The config.json to use. If unspecified, use the config compiled into URL Cleaner
     #[arg(short      , long)] config: Option<PathBuf>,

src/types/advanced_requests.rs

@@ -27,8 +27,8 @@ pub struct RequestConfig {
     /// The HTTP method to use. Defaults to [`Method::GET`].
     #[serde(deserialize_with = "deserialize_method", serialize_with = "serialize_method", default = "get_get")]
     pub method: Method,
-    /// The headers to send in the request in addition to the default headers provided by [`Params::default_http_headers`].
-    /// Defaults to an emprty [`HeaderMap`].
+    /// The headers to send in the request in addition to the default headers provided by [`Params::http_client_config`] and [`Self::client_config_diff`].
+    /// Defaults to an empty [`HeaderMap`].
     #[serde(with = "headermap")]
     #[serde(default)]
     pub headers: HeaderMap,
@@ -94,6 +94,7 @@ impl RequestConfig {
     /// Makes a [`reqwest::blocking::RequestBuilder`].
     /// # Errors
     /// If the call to [`Params::http_client`] returns an error, that error is returned.
+    /// 
     /// If the call to [`RequestBody::apply`] returns an error, that error is returned.
     pub fn make(&self, url: &Url, params: &Params) -> Result<reqwest::blocking::RequestBuilder, RequestConfigError> {
         let mut ret=params.http_client(self.client_config_diff.as_ref())?.request(self.method.clone(), match self.url {Some(ref source) => Url::parse(get_string!(source, url, params, RequestConfigError))?, None => url.clone()});
@@ -106,7 +107,9 @@ impl RequestConfig {
     /// Sends the request then uses [`Self::response_handler`] to get a [`String`] from the [`reqwest::blocking::Response`].
     /// # Errors
     /// If the call to [`Self::make`] returns an error, that error is returned.
+    /// 
     /// If the call to [`reqwest::blocking::RequestBuilder::send`] returns an error, that error is returned.
+    /// 
     /// If the call to [`ResponseHandler`] returns an error, that error is returned.
     pub fn response(&self, url: &Url, params: &Params) -> Result<String, RequestConfigError> {
         Ok(self.response_handler.handle(self.make(url, params)?.send()?, url, params)?)
@@ -119,6 +122,7 @@ pub enum RequestBody {
     /// [`reqwest::blocking::RequestBuilder::body`].
     /// # Errors
     /// If the call to [`StringSource::get`] returns an error, that error is returned in a [`RequestBodyError::StringSourceError`].
+    /// 
     /// If the call to [`StringSource::get`] returns [`None`], returns the error [`RequestBodyError::StringSourceIsNone`]`.
     #[cfg(feature = "string-source")]
     Text(StringSource),
@@ -128,6 +132,7 @@ pub enum RequestBody {
     /// [`reqwest::blocking::RequestBuilder::form`].
     /// # Errors
     /// If a call to [`StringSource::get`] returns an error, that error is returned in a [`RequestBodyError::StringSourceError`].
+    /// 
     /// If a call to [`StringSource::get`] returns [`None`], returns the error [`RequestBodyError::StringSourceIsNone`]`.
     #[cfg(feature = "string-source")]
     Form(HashMap<String, StringSource>),
@@ -219,7 +224,7 @@ pub enum ResponseHandlerError {
     ToStrError(#[from] reqwest::header::ToStrError),
     /// Returned when the requested cookie is not found.
     #[error("The requested cookie was not found.")]
-    CookieNotFound,
+    CookieNotFound
 }
 
 #[cfg(feature = "string-source")]