Issue 382/ support X-Robots-Tag as a typed http header XRobotsTag

rama-http-types/src/lib.rs

@@ -131,6 +131,12 @@ pub mod header {
         "x-real-ip",
     ];
 
+    // non-std web-crawler info headers
+    //
+    // More information at
+    // <https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Robots-Tag>.
+    static_header!["x-robots-tag"];
+
     /// Static Header Value that is can be used as `User-Agent` or `Server` header.
     pub static RAMA_ID_HEADER_VALUE: HeaderValue = HeaderValue::from_static(
         const_format::formatcp!("{}/{}", rama_utils::info::NAME, rama_utils::info::VERSION),

rama-http/Cargo.toml

@@ -30,6 +30,7 @@ async-compression = { workspace = true, features = [
 base64 = { workspace = true }
 bitflags = { workspace = true }
 bytes = { workspace = true }
+chrono = "0.4.39"
 const_format = { workspace = true }
 futures-lite = { workspace = true }
 headers = { workspace = true }

rama-http/src/headers/mod.rs

@@ -102,4 +102,7 @@ pub mod authorization {
 pub use ::rama_http_types::headers::HeaderExt;
 
 pub(crate) mod util;
+
+pub mod x_robots_tag;
+
 pub use util::quality_value::{Quality, QualityValue};

rama-http/src/headers/util/csv.rs

@@ -11,24 +11,28 @@ use crate::HeaderValue;
 pub(crate) fn from_comma_delimited<'i, I, T, E>(values: &mut I) -> Result<E, Error>
 where
     I: Iterator<Item = &'i HeaderValue>,
-    T: ::std::str::FromStr,
-    E: ::std::iter::FromIterator<T>,
+    T: std::str::FromStr,
+    E: FromIterator<T>,
 {
     values
         .flat_map(|value| {
-            value.to_str().into_iter().flat_map(|string| {
-                string
-                    .split(',')
-                    .filter_map(|x| match x.trim() {
-                        "" => None,
-                        y => Some(y),
-                    })
-                    .map(|x| x.parse().map_err(|_| Error::invalid()))
-            })
+            value
+                .to_str()
+                .into_iter()
+                .flat_map(|string| split_csv_str(string))
         })
         .collect()
 }
 
+pub(crate) fn split_csv_str<T: std::str::FromStr>(
+    string: &str,
+) -> impl Iterator<Item = Result<T, Error>> + use<'_, T> {
+    string.split(',').filter_map(|x| match x.trim() {
+        "" => None,
+        y => Some(y.parse().map_err(|_| Error::invalid())),
+    })
+}
+
 /// Format an array into a comma-delimited string.
 pub(crate) fn fmt_comma_delimited<T: fmt::Display>(
     f: &mut fmt::Formatter,

rama-http/src/headers/util/mod.rs

@@ -1,3 +1,5 @@
 pub(crate) mod csv;
 /// Internal utility functions for headers.
 pub(crate) mod quality_value;
+
+pub(crate) mod value_string;

rama-http/src/headers/util/value_string.rs

@@ -0,0 +1,81 @@
+use std::{
+    fmt,
+    str::{self, FromStr},
+};
+
+use bytes::Bytes;
+use http::header::HeaderValue;
+
+use crate::headers::Error;
+
+/// A value that is both a valid `HeaderValue` and `String`.
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct HeaderValueString {
+    /// Care must be taken to only set this value when it is also
+    /// a valid `String`, since `as_str` will convert to a `&str`
+    /// in an unchecked manner.
+    value: HeaderValue,
+}
+
+impl HeaderValueString {
+    pub(crate) fn from_val(val: &HeaderValue) -> Result<Self, Error> {
+        if val.to_str().is_ok() {
+            Ok(HeaderValueString { value: val.clone() })
+        } else {
+            Err(Error::invalid())
+        }
+    }
+
+    pub(crate) fn from_string(src: String) -> Option<Self> {
+        // A valid `str` (the argument)...
+        let bytes = Bytes::from(src);
+        HeaderValue::from_maybe_shared(bytes)
+            .ok()
+            .map(|value| HeaderValueString { value })
+    }
+
+    pub(crate) fn from_static(src: &'static str) -> HeaderValueString {
+        // A valid `str` (the argument)...
+        HeaderValueString {
+            value: HeaderValue::from_static(src),
+        }
+    }
+
+    pub(crate) fn as_str(&self) -> &str {
+        // HeaderValueString is only created from HeaderValues
+        // that have validated they are also UTF-8 strings.
+        unsafe { str::from_utf8_unchecked(self.value.as_bytes()) }
+    }
+}
+
+impl fmt::Debug for HeaderValueString {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Debug::fmt(self.as_str(), f)
+    }
+}
+
+impl fmt::Display for HeaderValueString {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Display::fmt(self.as_str(), f)
+    }
+}
+
+impl<'a> From<&'a HeaderValueString> for HeaderValue {
+    fn from(src: &'a HeaderValueString) -> HeaderValue {
+        src.value.clone()
+    }
+}
+
+#[derive(Debug)]
+pub struct FromStrError(());
+
+impl FromStr for HeaderValueString {
+    type Err = FromStrError;
+
+    fn from_str(src: &str) -> Result<Self, Self::Err> {
+        // A valid `str` (the argument)...
+        src.parse()
+            .map(|value| HeaderValueString { value })
+            .map_err(|_| FromStrError(()))
+    }
+}

rama-http/src/headers/x_robots_tag/custom_rule.rs

@@ -0,0 +1,31 @@
+use crate::headers::util::value_string::{FromStrError, HeaderValueString};
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct CustomRule {
+    key: HeaderValueString,
+    value: Option<HeaderValueString>,
+}
+
+impl CustomRule {
+    pub fn new(key: &str) -> Result<Self, FromStrError> {
+        Ok(Self {
+            key: key.parse()?,
+            value: None,
+        })
+    }
+
+    pub fn with_value(key: &str, value: &str) -> Result<Self, FromStrError> {
+        Ok(Self {
+            key: key.parse()?,
+            value: Some(value.parse()?),
+        })
+    }
+
+    pub fn key(&self) -> &HeaderValueString {
+        &self.key
+    }
+
+    pub fn value(&self) -> Option<&HeaderValueString> {
+        self.value.as_ref()
+    }
+}

rama-http/src/headers/x_robots_tag/max_image_preview_setting.rs

@@ -0,0 +1,35 @@
+use rama_core::error::OpaqueError;
+use std::fmt::Formatter;
+use std::str::FromStr;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum MaxImagePreviewSetting {
+    None,
+    Standard,
+    Large,
+}
+
+impl std::fmt::Display for MaxImagePreviewSetting {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            MaxImagePreviewSetting::None => write!(f, "none"),
+            MaxImagePreviewSetting::Standard => write!(f, "standard"),
+            MaxImagePreviewSetting::Large => write!(f, "large"),
+        }
+    }
+}
+
+impl FromStr for MaxImagePreviewSetting {
+    type Err = OpaqueError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.to_lowercase().trim() {
+            "none" => Ok(MaxImagePreviewSetting::None),
+            "standard" => Ok(MaxImagePreviewSetting::Standard),
+            "large" => Ok(MaxImagePreviewSetting::Large),
+            _ => Err(OpaqueError::from_display(
+                "failed to parse MaxImagePreviewSetting",
+            )),
+        }
+    }
+}

rama-http/src/headers/x_robots_tag/mod.rs

@@ -0,0 +1,11 @@
+pub mod robots_tag;
+pub use robots_tag::RobotsTag;
+
+pub mod robots_tag_builder;
+pub use robots_tag_builder::RobotsTagBuilder;
+
+pub mod max_image_preview_setting;
+pub use max_image_preview_setting::MaxImagePreviewSetting;
+
+pub mod custom_rule;
+pub use custom_rule::CustomRule;