Add a mode to handle "pretty URLs", i.e. URIs without .html extension

dscho · dscho · commit db7741832788 · 2024-05-13T00:51:20.000+02:00
In many circumstances (GitHub Pages, Apache configured with MultiViews,
etc), web servers process URIs by appending the `.html` file extension
when no file is found at the path specified by the URI but a `.html`
file corresponding to that path _is_ found.

To allow Lychee to use the fast, offline method of checking such files
locally via the `file://` scheme, let's handle this scenario gracefully
by adding the `--auto-append-html-fileext` option.

Signed-off-by: Johannes Schindelin &lt;johannes.schindelin@gmx.de&gt;
diff --git a/README.md b/README.md
@@ -389,6 +389,10 @@ Options:
       --remap <REMAP>
           Remap URI matching pattern to different URI
 
+      --auto-append-html-fileext
+          Automatically append `.html` to `file://` URLs when no file could be found
+          at the specified `path`.
+
       --header <HEADER>
           Custom request header
 
diff --git a/fixtures/pretty-urls/index.html b/fixtures/pretty-urls/index.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>For Testing pretty URLs</title>
+  </head>
+  <body>
+    <a href="other">other</a>
+  </body>
+</html>
diff --git a/fixtures/pretty-urls/other.html b/fixtures/pretty-urls/other.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>For Testing pretty URLs</title>
+  </head>
+  <body>
+    <a href="index">index</a>
+  </body>
+</html>
diff --git a/lychee-bin/src/client.rs b/lychee-bin/src/client.rs
@@ -75,6 +75,7 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         .require_https(cfg.require_https)
         .cookie_jar(cookie_jar.cloned())
         .include_fragments(cfg.include_fragments)
+        .auto_append_html_fileext(cfg.auto_append_html_fileext)
         .build()
         .client()
         .context("Failed to create request client")
diff --git a/lychee-bin/src/options.rs b/lychee-bin/src/options.rs
@@ -300,6 +300,15 @@ pub(crate) struct Config {
     #[arg(long)]
     pub(crate) remap: Vec<String>,
 
+    /// Automatically append `.html` to `file://` URIs as needed
+    #[serde(default)]
+    #[arg(
+        long,
+        long_help = "Automatically append `.html` to `file://` URLs when no file could be found
+at the specified `path`."
+    )]
+    pub(crate) auto_append_html_fileext: bool,
+
     /// Custom request header
     #[arg(long)]
     #[serde(default)]
@@ -439,6 +448,7 @@ impl Config {
             exclude_loopback: false;
             exclude_mail: false;
             remap: Vec::<String>::new();
+            auto_append_html_fileext: false;
             header: Vec::<String>::new();
             timeout: DEFAULT_TIMEOUT_SECS;
             retry_wait_time: DEFAULT_RETRY_WAIT_TIME_SECS;
diff --git a/lychee-bin/tests/cli.rs b/lychee-bin/tests/cli.rs
@@ -1556,4 +1556,17 @@ mod cli {
             // 3 failures because of missing fragments
             .stdout(contains("3 Errors"));
     }
+
+    #[test]
+    fn test_pretty_urls() {
+        let mut cmd = main_command();
+        let input = fixtures_path().join("pretty-urls");
+
+        cmd.arg("--verbose")
+            .arg("--auto-append-html-fileext")
+            .arg(input)
+            .assert()
+            .success()
+            .stdout(contains("0 Errors"));
+    }
 }
diff --git a/lychee-lib/src/client.rs b/lychee-lib/src/client.rs
@@ -95,6 +95,9 @@ pub struct ClientBuilder {
     /// make sure rules don't conflict with each other.
     remaps: Option<Remaps>,
 
+    /// Automatically append `.html` file extensions to `file://` URIs as needed
+    auto_append_html_fileext: bool,
+
     /// Links matching this set of regular expressions are **always** checked.
     ///
     /// This has higher precedence over [`ClientBuilder::excludes`], **but**
@@ -384,6 +387,7 @@ impl ClientBuilder {
             reqwest_client,
             github_client,
             remaps: self.remaps,
+            auto_append_html_fileext: self.auto_append_html_fileext,
             filter,
             max_retries: self.max_retries,
             retry_wait_time: self.retry_wait_time,
@@ -412,6 +416,9 @@ pub struct Client {
     /// Optional remapping rules for URIs matching pattern.
     remaps: Option<Remaps>,
 
+    /// Automatically append `.html` file extensions to `file://` URIs as needed
+    auto_append_html_fileext: bool,
+
     /// Rules to decided whether each link should be checked or ignored.
     filter: Filter,
 
@@ -654,14 +661,25 @@ impl Client {
         let Ok(path) = uri.url.to_file_path() else {
             return ErrorKind::InvalidFilePath(uri.clone()).into();
         };
-        if !path.exists() {
-            return ErrorKind::InvalidFilePath(uri.clone()).into();
-        }
-        if self.include_fragments {
-            self.check_fragment(&path, uri).await
+        if path.exists() {
+            if self.include_fragments {
+                return self.check_fragment(&path, uri).await;
+            }
         } else {
-            Status::Ok(StatusCode::OK)
+            // if the path does not end in `.html`, try to append it
+            if !self.auto_append_html_fileext || path.extension().is_some() {
+                return ErrorKind::InvalidFilePath(uri.clone()).into();
+            }
+            // append `.html` and try again
+            let mut path_buf = path.clone();
+            path_buf.set_extension("html");
+            if !path_buf.exists() {
+                return ErrorKind::InvalidFilePath(uri.clone()).into();
+            } else if self.include_fragments {
+                return self.check_fragment(&path_buf, uri).await;
+            }
         }
+        Status::Ok(StatusCode::OK)
     }
 
     /// Checks a `file` URI's fragment.
