From 9e57129cff1d03b96974f88817413645103bcda8 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Wed, 8 Feb 2023 15:28:34 +0000 Subject: [PATCH 01/56] add initial --- src/services/webdav/backend.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index a8953c341ba2..4e52ab1c776e 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -270,6 +270,19 @@ impl Accessor for WebdavBackend { ma } + async fn list(&self, path: &str, args: OpList) -> Result<(RpList, ObjectPager)> { + let (_, _) = (path, args); + + let empoty_string = AsyncBody::from("".into()); + + let resp = self.webdav_put(path, None, "application/xml".into(), empoty_string); + + Err(Error::new( + ErrorKind::Unsupported, + "operation is not supported", + )) + } + async fn create(&self, path: &str, _: OpCreate) -> Result { let resp = self .webdav_put(path, Some(0), None, AsyncBody::Empty) From 253a7018ccf927fd0511c0cfeb0a2c519870cfa6 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 9 Feb 2023 07:11:12 -0800 Subject: [PATCH 02/56] add list response --- src/services/webdav/list_response.rs | 133 +++++++++++++++++++++++++++ src/services/webdav/mod.rs | 1 + 2 files changed, 134 insertions(+) create mode 100644 src/services/webdav/list_response.rs diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs new file mode 100644 index 000000000000..9a704520737d --- /dev/null +++ b/src/services/webdav/list_response.rs @@ -0,0 +1,133 @@ +use serde::Deserialize; + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +struct Multistatus { + response: Vec, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +struct ListOpResponse { + href: String, + propstat: Propstat, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +struct Propstat { + prop: Prop, + status: String, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +struct Prop { + displayname: String, + getlastmodified: String, + resourcetype: ResourceType, + lockdiscovery: (), + supportedlock: SupportedLock, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +#[derive(PartialEq)] +enum ResourceType { + Collection, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +struct SupportedLock { + lockentry: LockEntry, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +struct LockEntry { + lockscope: LockScope, + locktype: LockType, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +#[derive(PartialEq)] +enum LockScope { + Exclusive, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "PascalCase")] +#[derive(PartialEq)] +enum LockType { + Write, +} + +#[cfg(test)] +mod tests { + use super::*; + use quick_xml::de::from_str; + + #[test] + fn test_lockentry() { + let xml = r#" + + + "#; + + let lockentry = from_str::(xml).unwrap(); + assert_eq!(lockentry.lockscope, LockScope::Exclusive); + assert_eq!(lockentry.locktype, LockType::Write); + } + + #[test] + fn test_supportedlock() { + let xml = r#" + + + + + "#; + + let supportedlock = from_str::(xml).unwrap(); + assert_eq!(supportedlock.lockentry.lockscope, LockScope::Exclusive); + assert_eq!(supportedlock.lockentry.locktype, LockType::Write); + } + + #[test] + fn test_propstat() { + let xml = r#" + + / + Tue, 07 Feb 2023 06:39:47 GMT + + + + + + + + + + HTTP/1.1 200 OK + "#; + + let propstat = from_str::(xml).unwrap(); + assert_eq!(propstat.prop.displayname, "/"); + assert_eq!( + propstat.prop.getlastmodified, + "Tue, 07 Feb 2023 06:39:47 GMT" + ); + assert_eq!(propstat.prop.resourcetype, ResourceType::Collection); + assert_eq!( + propstat.prop.supportedlock.lockentry.lockscope, + LockScope::Exclusive + ); + assert_eq!( + propstat.prop.supportedlock.lockentry.locktype, + LockType::Write + ); + assert_eq!(propstat.status, "HTTP/1.1 200 OK"); + } +} diff --git a/src/services/webdav/mod.rs b/src/services/webdav/mod.rs index ba8c6e5b3b39..1f44d4b5d9b2 100644 --- a/src/services/webdav/mod.rs +++ b/src/services/webdav/mod.rs @@ -16,3 +16,4 @@ mod backend; pub use backend::WebdavBuilder as Webdav; mod error; +mod list_response; From 51f00873f8d7daca629042cac3770bce8286a933 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 08:42:25 -0800 Subject: [PATCH 03/56] add webdav list_response parser --- src/services/webdav/backend.rs | 18 +- src/services/webdav/list_response.rs | 352 +++++++++++++++++++++++---- 2 files changed, 316 insertions(+), 54 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 4e52ab1c776e..b140d0c44860 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -270,18 +270,18 @@ impl Accessor for WebdavBackend { ma } - async fn list(&self, path: &str, args: OpList) -> Result<(RpList, ObjectPager)> { - let (_, _) = (path, args); + // async fn list(&self, path: &str, args: OpList) -> Result<(RpList, ObjectPager)> { + // let (_, _) = (path, args); - let empoty_string = AsyncBody::from("".into()); + // let empoty_string = AsyncBody::from("".into()); - let resp = self.webdav_put(path, None, "application/xml".into(), empoty_string); + // let resp = self.webdav_put(path, None, "application/xml".into(), empoty_string); - Err(Error::new( - ErrorKind::Unsupported, - "operation is not supported", - )) - } + // Err(Error::new( + // ErrorKind::Unsupported, + // "operation is not supported", + // )) + // } async fn create(&self, path: &str, _: OpCreate) -> Result { let resp = self diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 9a704520737d..32cc4af786f0 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -1,74 +1,84 @@ use serde::Deserialize; -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] +#[derive(Deserialize, Debug, PartialEq)] +#[serde(rename_all = "lowercase")] +pub struct LockEntry { + pub lockscope: LockScopeContainer, + pub locktype: LockTypeContainer, +} + +#[derive(Deserialize, Debug, PartialEq)] +pub struct LockScopeContainer { + #[serde(rename = "$value")] + pub value: LockScope, +} + +#[derive(Deserialize, Debug, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum LockScope { + Exclusive, +} + +#[derive(Deserialize, Debug, PartialEq)] +pub struct LockTypeContainer { + #[serde(rename = "$value")] + pub value: LockType, +} + +#[derive(Deserialize, Debug, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum LockType { + Write, +} + +#[derive(Deserialize, Debug, PartialEq)] struct Multistatus { response: Vec, } -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] +#[derive(Deserialize, Debug, PartialEq)] struct ListOpResponse { href: String, propstat: Propstat, } -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] +#[derive(Deserialize, Debug, PartialEq)] struct Propstat { prop: Prop, status: String, } -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] +#[derive(Deserialize, Debug, PartialEq)] struct Prop { displayname: String, getlastmodified: String, - resourcetype: ResourceType, + resourcetype: ResourceTypeContainer, lockdiscovery: (), supportedlock: SupportedLock, } -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -#[derive(PartialEq)] +#[derive(Deserialize, Debug, PartialEq)] +struct ResourceTypeContainer { + #[serde(rename = "$value")] + value: Option, +} + +#[derive(Deserialize, Debug, PartialEq)] +#[serde(rename_all = "lowercase")] enum ResourceType { Collection, } -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] +#[derive(Deserialize, Debug, PartialEq)] + struct SupportedLock { lockentry: LockEntry, } -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -struct LockEntry { - lockscope: LockScope, - locktype: LockType, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -#[derive(PartialEq)] -enum LockScope { - Exclusive, -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "PascalCase")] -#[derive(PartialEq)] -enum LockType { - Write, -} - #[cfg(test)] mod tests { use super::*; use quick_xml::de::from_str; - #[test] fn test_lockentry() { let xml = r#" @@ -77,8 +87,8 @@ mod tests { "#; let lockentry = from_str::(xml).unwrap(); - assert_eq!(lockentry.lockscope, LockScope::Exclusive); - assert_eq!(lockentry.locktype, LockType::Write); + assert_eq!(lockentry.lockscope.value, LockScope::Exclusive); + assert_eq!(lockentry.locktype.value, LockType::Write); } #[test] @@ -91,8 +101,11 @@ mod tests { "#; let supportedlock = from_str::(xml).unwrap(); - assert_eq!(supportedlock.lockentry.lockscope, LockScope::Exclusive); - assert_eq!(supportedlock.lockentry.locktype, LockType::Write); + assert_eq!( + supportedlock.lockentry.lockscope.value, + LockScope::Exclusive + ); + assert_eq!(supportedlock.lockentry.locktype.value, LockType::Write); } #[test] @@ -100,7 +113,7 @@ mod tests { let xml = r#" / - Tue, 07 Feb 2023 06:39:47 GMT + Tue, 01 May 2022 06:39:47 GMT @@ -117,17 +130,266 @@ mod tests { assert_eq!(propstat.prop.displayname, "/"); assert_eq!( propstat.prop.getlastmodified, - "Tue, 07 Feb 2023 06:39:47 GMT" + "Tue, 01 May 2022 06:39:47 GMT" ); - assert_eq!(propstat.prop.resourcetype, ResourceType::Collection); assert_eq!( - propstat.prop.supportedlock.lockentry.lockscope, + propstat.prop.resourcetype.value.unwrap(), + ResourceType::Collection + ); + assert_eq!( + propstat.prop.supportedlock.lockentry.lockscope.value, LockScope::Exclusive ); assert_eq!( - propstat.prop.supportedlock.lockentry.locktype, + propstat.prop.supportedlock.lockentry.locktype.value, LockType::Write ); assert_eq!(propstat.status, "HTTP/1.1 200 OK"); } + + #[test] + fn test_response_simple() { + let xml = r#" + / + + + / + Tue, 01 May 2022 06:39:47 GMT + + + + + + + + + + HTTP/1.1 200 OK + + "#; + + let response = from_str::(xml).unwrap(); + assert_eq!(response.href, "/"); + assert_eq!(response.propstat.prop.displayname, "/"); + assert_eq!( + response.propstat.prop.getlastmodified, + "Tue, 01 May 2022 06:39:47 GMT" + ); + assert_eq!( + response.propstat.prop.resourcetype.value.unwrap(), + ResourceType::Collection + ); + assert_eq!( + response + .propstat + .prop + .supportedlock + .lockentry + .lockscope + .value, + LockScope::Exclusive + ); + assert_eq!( + response + .propstat + .prop + .supportedlock + .lockentry + .locktype + .value, + LockType::Write + ); + assert_eq!(response.propstat.status, "HTTP/1.1 200 OK"); + } + + #[test] + fn test_response_file() { + let xml = r#" + /test_file + + + test_file + 1 + Tue, 07 May 2022 05:52:22 GMT + + + + + + + + + + + + + + HTTP/1.1 200 OK + + "#; + + let response = from_str::(xml).unwrap(); + assert_eq!(response.href, "/test_file"); + assert_eq!(response.propstat.prop.displayname, "test_file"); + assert_eq!( + response.propstat.prop.getlastmodified, + "Tue, 07 May 2022 05:52:22 GMT" + ); + assert_eq!(response.propstat.prop.resourcetype.value, None); + + assert_eq!( + response + .propstat + .prop + .supportedlock + .lockentry + .lockscope + .value, + LockScope::Exclusive + ); + assert_eq!( + response + .propstat + .prop + .supportedlock + .lockentry + .locktype + .value, + LockType::Write + ); + assert_eq!(response.propstat.status, "HTTP/1.1 200 OK"); + } + + #[test] + fn test_with_multiple_items_simple() { + let xml = r#" + + / + + + / + Tue, 01 May 2022 06:39:47 GMT + + + + + + + + + + HTTP/1.1 200 OK + + + + / + + + / + Tue, 01 May 2022 06:39:47 GMT + + + + + + + + + + HTTP/1.1 200 OK + + + "#; + + let multistatus = from_str::(xml).unwrap(); + assert_eq!(multistatus.response.len(), 2); + assert_eq!(multistatus.response[0].href, "/"); + assert_eq!(multistatus.response[0].propstat.prop.displayname, "/"); + assert_eq!( + multistatus.response[0].propstat.prop.getlastmodified, + "Tue, 01 May 2022 06:39:47 GMT" + ); + } + + #[test] + fn test_with_multiple_items_mixed() { + let xml = r#" + + + / + + + / + Tue, 07 May 2022 06:39:47 GMT + + + + + + + + + + + + + + + + HTTP/1.1 200 OK + + + + /testdir/ + + + testdir + Tue, 07 May 2022 06:40:10 GMT + + + + + + + + + + + + + + + + HTTP/1.1 200 OK + + + + /test_file + + + test_file + 1 + Tue, 07 May 2022 05:52:22 GMT + + + + + + + + + + + + + + HTTP/1.1 200 OK + + + "#; + + let multistatus = from_str::(xml).unwrap(); + + assert_eq!(multistatus.response.len(), 3); + } } From 2b560148b9d8340b5e18012575c256f2c2858ed2 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 08:54:55 -0800 Subject: [PATCH 04/56] add more response --- src/services/webdav/list_response.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 32cc4af786f0..0a67232a1d9c 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -391,5 +391,28 @@ mod tests { let multistatus = from_str::(xml).unwrap(); assert_eq!(multistatus.response.len(), 3); + let first_response = &multistatus.response[0]; + assert_eq!(first_response.href, "/"); + assert_eq!(first_response.propstat.prop.displayname, "/"); + assert_eq!( + first_response.propstat.prop.getlastmodified, + "Tue, 07 May 2022 06:39:47 GMT" + ); + + let second_response = &multistatus.response[1]; + assert_eq!(second_response.href, "/testdir/"); + assert_eq!(second_response.propstat.prop.displayname, "testdir"); + assert_eq!( + second_response.propstat.prop.getlastmodified, + "Tue, 07 May 2022 06:40:10 GMT" + ); + + let third_response = &multistatus.response[2]; + assert_eq!(third_response.href, "/test_file"); + assert_eq!(third_response.propstat.prop.displayname, "test_file"); + assert_eq!( + third_response.propstat.prop.getlastmodified, + "Tue, 07 May 2022 05:52:22 GMT" + ); } } From 9660bfbdbd7531b146f726baa4e4bc070e3906a1 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 10:47:58 -0800 Subject: [PATCH 05/56] RFR: making PUT more generic --- src/services/webdav/backend.rs | 35 ++++++++++++++++++++++-- src/services/webdav/body_request_type.rs | 14 ++++++++++ src/services/webdav/mod.rs | 1 + 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 src/services/webdav/body_request_type.rs diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index b140d0c44860..77c15f81bfeb 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -27,6 +27,8 @@ use http::Response; use http::StatusCode; use log::debug; +use super::body_request_type; +use super::body_request_type::BodyRequestType; use super::error::parse_error; use crate::ops::*; use crate::raw::*; @@ -395,8 +397,9 @@ impl WebdavBackend { self.client.send_async(req).await } - async fn webdav_put( + async fn webdav_custom_request( &self, + body_request_type: BodyRequestType, path: &str, size: Option, content_type: Option<&str>, @@ -405,8 +408,12 @@ impl WebdavBackend { let p = build_abs_path(&self.root, path); let url = format!("{}/{}", self.endpoint, percent_encode_path(&p)); - - let mut req = Request::put(&url).header(AUTHORIZATION, &self.authorization); + let body_type_string = body_request_type.to_string(); + let method: &str = body_type_string.as_str(); + let mut req = Request::builder() + .method(method) + .uri(&url) + .header(AUTHORIZATION, &self.authorization); if let Some(size) = size { req = req.header(CONTENT_LENGTH, size) @@ -422,6 +429,28 @@ impl WebdavBackend { self.client.send_async(req).await } + async fn webdav_put( + &self, + path: &str, + size: Option, + content_type: Option<&str>, + body: AsyncBody, + ) -> Result> { + self.webdav_custom_request(BodyRequestType::PUT, path, size, content_type, body) + .await + } + + async fn webdav_propfind( + &self, + path: &str, + size: Option, + content_type: Option<&str>, + body: AsyncBody, + ) -> Result> { + self.webdav_custom_request(BodyRequestType::PROPFIND, path, size, content_type, body) + .await + } + async fn webdav_head(&self, path: &str) -> Result> { let p = build_rooted_abs_path(&self.root, path); diff --git a/src/services/webdav/body_request_type.rs b/src/services/webdav/body_request_type.rs new file mode 100644 index 000000000000..f4295177e380 --- /dev/null +++ b/src/services/webdav/body_request_type.rs @@ -0,0 +1,14 @@ +#[derive(Debug, Clone)] +pub enum BodyRequestType { + PUT, + PROPFIND, +} + +impl ToString for BodyRequestType { + fn to_string(&self) -> String { + match self { + BodyRequestType::PUT => "PUT".to_string(), + BodyRequestType::PROPFIND => "PROPFIND".to_string(), + } + } +} diff --git a/src/services/webdav/mod.rs b/src/services/webdav/mod.rs index 1f44d4b5d9b2..8e971cf21b1b 100644 --- a/src/services/webdav/mod.rs +++ b/src/services/webdav/mod.rs @@ -15,5 +15,6 @@ mod backend; pub use backend::WebdavBuilder as Webdav; +mod body_request_type; mod error; mod list_response; From 71241409c48d0fd84e0b5310434d74b23502f630 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 11:32:02 -0800 Subject: [PATCH 06/56] progress implementatin --- src/services/webdav/backend.rs | 40 +++++++++++++++++++++------- src/services/webdav/list_response.rs | 4 +-- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 77c15f81bfeb..3f28d06debaa 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -19,6 +19,7 @@ use std::fmt::Formatter; use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; +use bytes::Buf; use http::header::AUTHORIZATION; use http::header::CONTENT_LENGTH; use http::header::CONTENT_TYPE; @@ -27,9 +28,9 @@ use http::Response; use http::StatusCode; use log::debug; -use super::body_request_type; use super::body_request_type::BodyRequestType; use super::error::parse_error; +use super::list_response::Multistatus; use crate::ops::*; use crate::raw::*; use crate::*; @@ -272,18 +273,37 @@ impl Accessor for WebdavBackend { ma } - // async fn list(&self, path: &str, args: OpList) -> Result<(RpList, ObjectPager)> { - // let (_, _) = (path, args); + async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> { + let _ = args; - // let empoty_string = AsyncBody::from("".into()); + let all_prop_xml_body = r#" + + + + "#; - // let resp = self.webdav_put(path, None, "application/xml".into(), empoty_string); + let async_body = AsyncBody::Bytes(bytes::Bytes::from(all_prop_xml_body)); - // Err(Error::new( - // ErrorKind::Unsupported, - // "operation is not supported", - // )) - // } + let resp = self + .webdav_put(path, None, "application/xml".into(), async_body) + .await?; + + let status = resp.status(); + + match status { + StatusCode::OK | StatusCode::MULTI_STATUS => { + let bs = resp.into_body().bytes().await?; + let result: Multistatus = + quick_xml::de::from_reader(bs.reader()).map_err(|err| { + Error::new(ErrorKind::Unexpected, &err.to_string()) + .with_context("service", Scheme::Webdav) + })?; + + let mut entries = Vec::new(); + } + _ => Err(parse_error(resp).await?), // TODO: handle error gracefully + } + } async fn create(&self, path: &str, _: OpCreate) -> Result { let resp = self diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 0a67232a1d9c..261f1b12dabf 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -32,12 +32,12 @@ pub enum LockType { } #[derive(Deserialize, Debug, PartialEq)] -struct Multistatus { +pub struct Multistatus { response: Vec, } #[derive(Deserialize, Debug, PartialEq)] -struct ListOpResponse { +pub struct ListOpResponse { href: String, propstat: Propstat, } From a587aece93776ecbbbd886cdd6d88e0ed275482f Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 11:59:13 -0800 Subject: [PATCH 07/56] add dir_stream --- src/services/webdav/dir_stream.rs | 65 +++++++++++++++++++++++++++++++ src/services/webdav/mod.rs | 1 + 2 files changed, 66 insertions(+) create mode 100644 src/services/webdav/dir_stream.rs diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs new file mode 100644 index 000000000000..260413eb7314 --- /dev/null +++ b/src/services/webdav/dir_stream.rs @@ -0,0 +1,65 @@ +use crate::Result; +use crate::{ + raw::{normalize_path, output}, + ObjectMetadata, ObjectMode, +}; +use async_trait::async_trait; + +use super::{backend::WebdavBackend, list_response::Multistatus}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +pub struct DirPager { + root: PathBuf, + + size: usize, + multistates: Multistatus, +} + +impl DirPager { + pub fn new(root: &Path, multistates: Multistatus, limit: Option) -> Self { + Self { + root: root.to_owned(), + size: limit.unwrap_or(1000), + multistates: multistates, + } + } +} + +#[async_trait] +impl output::Page for DirPager { + async fn next_page(&mut self) -> Result>> { + let mut oes: Vec = Vec::with_capacity(self.size); + + for _ in 0..self.size { + let de = match self.multistates.response.into_iter().next() { + Some(de) => { + let path = PathBuf::from(de.href); + + let rel_path = normalize_path( + &path + .strip_prefix(&self.root) + .expect("cannot fail because the prefix is iterated") + .to_string_lossy() + .replace('\\', "/"), + ); + + let entry = if de.propstat.prop.resourcetype.value + == Some(super::list_response::ResourceType::Collection) + { + output::Entry::new(&rel_path, ObjectMetadata::new(ObjectMode::DIR)) + } else { + output::Entry::new(&rel_path, ObjectMetadata::new(ObjectMode::FILE)) + }; + + oes.push(entry); + } + None => break, + }; + } + + Ok(if oes.is_empty() { None } else { Some(oes) }) + } +} diff --git a/src/services/webdav/mod.rs b/src/services/webdav/mod.rs index 8e971cf21b1b..0de33aefb3e0 100644 --- a/src/services/webdav/mod.rs +++ b/src/services/webdav/mod.rs @@ -16,5 +16,6 @@ mod backend; pub use backend::WebdavBuilder as Webdav; mod body_request_type; +mod dir_stream; mod error; mod list_response; From fc278a2ba0420ccc363c7528077c9cb9610ccb6a Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 12:00:41 -0800 Subject: [PATCH 08/56] make fields public --- src/services/webdav/list_response.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 261f1b12dabf..5b42cfd96c19 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -33,39 +33,39 @@ pub enum LockType { #[derive(Deserialize, Debug, PartialEq)] pub struct Multistatus { - response: Vec, + pub response: Vec, } #[derive(Deserialize, Debug, PartialEq)] pub struct ListOpResponse { - href: String, - propstat: Propstat, + pub href: String, + pub propstat: Propstat, } #[derive(Deserialize, Debug, PartialEq)] struct Propstat { - prop: Prop, - status: String, + pub prop: Prop, + pub status: String, } #[derive(Deserialize, Debug, PartialEq)] -struct Prop { - displayname: String, - getlastmodified: String, - resourcetype: ResourceTypeContainer, - lockdiscovery: (), - supportedlock: SupportedLock, +pub struct Prop { + pub displayname: String, + pub getlastmodified: String, + pub resourcetype: ResourceTypeContainer, + pub lockdiscovery: (), + pub supportedlock: SupportedLock, } #[derive(Deserialize, Debug, PartialEq)] -struct ResourceTypeContainer { +pub struct ResourceTypeContainer { #[serde(rename = "$value")] - value: Option, + pub value: Option, } #[derive(Deserialize, Debug, PartialEq)] #[serde(rename_all = "lowercase")] -enum ResourceType { +pub enum ResourceType { Collection, } From 651fd399bd5b913760e36de59300a3a7e80a0769 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 12:43:21 -0800 Subject: [PATCH 09/56] finish backend --- src/services/webdav/backend.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 3f28d06debaa..9ebc5b50dfa7 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; +use std::path::PathBuf; use async_trait::async_trait; use base64::engine::general_purpose; @@ -29,6 +30,7 @@ use http::StatusCode; use log::debug; use super::body_request_type::BodyRequestType; +use super::dir_stream::DirPager; use super::error::parse_error; use super::list_response::Multistatus; use crate::ops::*; @@ -260,7 +262,7 @@ impl Debug for WebdavBackend { impl Accessor for WebdavBackend { type Reader = IncomingAsyncBody; type BlockingReader = (); - type Pager = (); + type Pager = DirPager; type BlockingPager = (); fn metadata(&self) -> AccessorMetadata { @@ -274,8 +276,6 @@ impl Accessor for WebdavBackend { } async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> { - let _ = args; - let all_prop_xml_body = r#" @@ -283,11 +283,9 @@ impl Accessor for WebdavBackend { "#; let async_body = AsyncBody::Bytes(bytes::Bytes::from(all_prop_xml_body)); - let resp = self - .webdav_put(path, None, "application/xml".into(), async_body) + .webdav_propfind(path, None, "application/xml".into(), async_body) .await?; - let status = resp.status(); match status { @@ -299,7 +297,10 @@ impl Accessor for WebdavBackend { .with_context("service", Scheme::Webdav) })?; - let mut entries = Vec::new(); + Ok(( + RpList::default(), + DirPager::new(&PathBuf::from(self.root.clone()), result, args.limit()), + )) } _ => Err(parse_error(resp).await?), // TODO: handle error gracefully } From 9a7460a3e9d1e360a4f76bfc0d42590618637ab5 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 15:29:03 -0800 Subject: [PATCH 10/56] finish --- src/services/webdav/dir_stream.rs | 18 +++++++++--------- src/services/webdav/list_response.rs | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 260413eb7314..4a13d319b513 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -5,11 +5,8 @@ use crate::{ }; use async_trait::async_trait; -use super::{backend::WebdavBackend, list_response::Multistatus}; -use std::{ - path::{Path, PathBuf}, - sync::Arc, -}; +use super::list_response::Multistatus; +use std::path::{Path, PathBuf}; pub struct DirPager { root: PathBuf, @@ -33,10 +30,13 @@ impl output::Page for DirPager { async fn next_page(&mut self) -> Result>> { let mut oes: Vec = Vec::with_capacity(self.size); - for _ in 0..self.size { - let de = match self.multistates.response.into_iter().next() { + for i in 0..self.size { + if i >= self.multistates.response.len() { + break; + } + match self.multistates.response.get(0) { Some(de) => { - let path = PathBuf::from(de.href); + let path = PathBuf::from(de.href.clone()); let rel_path = normalize_path( &path @@ -57,7 +57,7 @@ impl output::Page for DirPager { oes.push(entry); } None => break, - }; + } } Ok(if oes.is_empty() { None } else { Some(oes) }) diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 5b42cfd96c19..35f7802a8d73 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -43,7 +43,7 @@ pub struct ListOpResponse { } #[derive(Deserialize, Debug, PartialEq)] -struct Propstat { +pub struct Propstat { pub prop: Prop, pub status: String, } @@ -71,7 +71,7 @@ pub enum ResourceType { #[derive(Deserialize, Debug, PartialEq)] -struct SupportedLock { +pub struct SupportedLock { lockentry: LockEntry, } From 35c713fd10b42fd208da5302978e0fc8e7e39c1c Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 15:30:53 -0800 Subject: [PATCH 11/56] rename to DirStream --- src/services/webdav/backend.rs | 6 +++--- src/services/webdav/dir_stream.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 9ebc5b50dfa7..872a7e3193c1 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -30,7 +30,7 @@ use http::StatusCode; use log::debug; use super::body_request_type::BodyRequestType; -use super::dir_stream::DirPager; +use super::dir_stream::DirStream; use super::error::parse_error; use super::list_response::Multistatus; use crate::ops::*; @@ -262,7 +262,7 @@ impl Debug for WebdavBackend { impl Accessor for WebdavBackend { type Reader = IncomingAsyncBody; type BlockingReader = (); - type Pager = DirPager; + type Pager = DirStream; type BlockingPager = (); fn metadata(&self) -> AccessorMetadata { @@ -299,7 +299,7 @@ impl Accessor for WebdavBackend { Ok(( RpList::default(), - DirPager::new(&PathBuf::from(self.root.clone()), result, args.limit()), + DirStream::new(&PathBuf::from(self.root.clone()), result, args.limit()), )) } _ => Err(parse_error(resp).await?), // TODO: handle error gracefully diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 4a13d319b513..6db9f23590e5 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -8,14 +8,14 @@ use async_trait::async_trait; use super::list_response::Multistatus; use std::path::{Path, PathBuf}; -pub struct DirPager { +pub struct DirStream { root: PathBuf, size: usize, multistates: Multistatus, } -impl DirPager { +impl DirStream { pub fn new(root: &Path, multistates: Multistatus, limit: Option) -> Self { Self { root: root.to_owned(), @@ -26,7 +26,7 @@ impl DirPager { } #[async_trait] -impl output::Page for DirPager { +impl output::Page for DirStream { async fn next_page(&mut self) -> Result>> { let mut oes: Vec = Vec::with_capacity(self.size); From 099049d21a8fc29d675ebd43d27bf11c8d0da0c1 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 15:49:05 -0800 Subject: [PATCH 12/56] add file header --- src/services/webdav/body_request_type.rs | 14 ++++++++++++++ src/services/webdav/dir_stream.rs | 14 ++++++++++++++ src/services/webdav/list_response.rs | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/src/services/webdav/body_request_type.rs b/src/services/webdav/body_request_type.rs index f4295177e380..a01b2ef570e1 100644 --- a/src/services/webdav/body_request_type.rs +++ b/src/services/webdav/body_request_type.rs @@ -1,3 +1,17 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #[derive(Debug, Clone)] pub enum BodyRequestType { PUT, diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 6db9f23590e5..20eef9756a00 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -1,3 +1,17 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use crate::Result; use crate::{ raw::{normalize_path, output}, diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 35f7802a8d73..5175fe551770 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -1,3 +1,17 @@ +// Copyright 2022 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use serde::Deserialize; #[derive(Deserialize, Debug, PartialEq)] From 975461ea7dfe2b36504798d25fb2c2a50de78448 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 16:10:24 -0800 Subject: [PATCH 13/56] fix lint --- src/services/webdav/backend.rs | 4 ++-- src/services/webdav/body_request_type.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 872a7e3193c1..d47ac1c96dda 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -457,7 +457,7 @@ impl WebdavBackend { content_type: Option<&str>, body: AsyncBody, ) -> Result> { - self.webdav_custom_request(BodyRequestType::PUT, path, size, content_type, body) + self.webdav_custom_request(BodyRequestType::Put, path, size, content_type, body) .await } @@ -468,7 +468,7 @@ impl WebdavBackend { content_type: Option<&str>, body: AsyncBody, ) -> Result> { - self.webdav_custom_request(BodyRequestType::PROPFIND, path, size, content_type, body) + self.webdav_custom_request(BodyRequestType::Propfind, path, size, content_type, body) .await } diff --git a/src/services/webdav/body_request_type.rs b/src/services/webdav/body_request_type.rs index a01b2ef570e1..8ac1c0e5f3c1 100644 --- a/src/services/webdav/body_request_type.rs +++ b/src/services/webdav/body_request_type.rs @@ -14,15 +14,15 @@ #[derive(Debug, Clone)] pub enum BodyRequestType { - PUT, - PROPFIND, + Put, + Propfind, } impl ToString for BodyRequestType { fn to_string(&self) -> String { match self { - BodyRequestType::PUT => "PUT".to_string(), - BodyRequestType::PROPFIND => "PROPFIND".to_string(), + BodyRequestType::Put => "PUT".to_string(), + BodyRequestType::Propfind => "PROPFIND".to_string(), } } } From d6fb97a325eb57ca845f83e6f77af07d113afedf Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 16:19:28 -0800 Subject: [PATCH 14/56] update comments --- src/services/webdav/backend.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index d47ac1c96dda..6b2f0555204b 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -45,7 +45,7 @@ use crate::*; /// /// - [x] read /// - [x] write -/// - [ ] list +/// - [x] list /// - [ ] ~~scan~~ /// - [ ] ~~presign~~ /// - [ ] ~~multipart~~ @@ -56,10 +56,6 @@ use crate::*; /// Bazel Remote Caching and Ccache HTTP Storage is also part of this service. /// Users can use `webdav` to connect those services. /// -/// # Status -/// -/// - `list` is not supported so far. -/// /// # Configuration /// /// - `endpoint`: set the endpoint for webdav From 9b80cf52f898c6f61ba4220115639c32a45d6039 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 20:52:19 -0800 Subject: [PATCH 15/56] set_capabilities --- src/services/webdav/backend.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 6b2f0555204b..e9af85d4f508 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -265,7 +265,9 @@ impl Accessor for WebdavBackend { let mut ma = AccessorMetadata::default(); ma.set_scheme(Scheme::Webdav) .set_root(&self.root) - .set_capabilities(AccessorCapability::Read | AccessorCapability::Write) + .set_capabilities( + AccessorCapability::Read | AccessorCapability::Write | AccessorCapability::List, + ) .set_hints(AccessorHint::ReadStreamable); ma From 04dd2e91682b408bf9d941d46d9f373ffe2374c6 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 22:11:55 -0800 Subject: [PATCH 16/56] address backend feedbacks --- src/services/webdav/backend.rs | 54 ++++++++++++------------ src/services/webdav/body_request_type.rs | 28 ------------ src/services/webdav/dir_stream.rs | 24 +++-------- src/services/webdav/mod.rs | 1 - 4 files changed, 32 insertions(+), 75 deletions(-) delete mode 100644 src/services/webdav/body_request_type.rs diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index e9af85d4f508..ab78b7e3a86f 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -12,11 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; -use std::fmt::Debug; -use std::fmt::Formatter; -use std::path::PathBuf; - use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; @@ -28,8 +23,10 @@ use http::Request; use http::Response; use http::StatusCode; use log::debug; +use std::collections::HashMap; +use std::fmt::Debug; +use std::fmt::Formatter; -use super::body_request_type::BodyRequestType; use super::dir_stream::DirStream; use super::error::parse_error; use super::list_response::Multistatus; @@ -297,7 +294,7 @@ impl Accessor for WebdavBackend { Ok(( RpList::default(), - DirStream::new(&PathBuf::from(self.root.clone()), result, args.limit()), + DirStream::new(self.root.clone(), result, args.limit()), )) } _ => Err(parse_error(resp).await?), // TODO: handle error gracefully @@ -416,9 +413,8 @@ impl WebdavBackend { self.client.send_async(req).await } - async fn webdav_custom_request( + async fn webdav_put( &self, - body_request_type: BodyRequestType, path: &str, size: Option, content_type: Option<&str>, @@ -427,12 +423,7 @@ impl WebdavBackend { let p = build_abs_path(&self.root, path); let url = format!("{}/{}", self.endpoint, percent_encode_path(&p)); - let body_type_string = body_request_type.to_string(); - let method: &str = body_type_string.as_str(); - let mut req = Request::builder() - .method(method) - .uri(&url) - .header(AUTHORIZATION, &self.authorization); + let mut req = Request::put(&url).header(AUTHORIZATION, &self.authorization); if let Some(size) = size { req = req.header(CONTENT_LENGTH, size) @@ -448,17 +439,6 @@ impl WebdavBackend { self.client.send_async(req).await } - async fn webdav_put( - &self, - path: &str, - size: Option, - content_type: Option<&str>, - body: AsyncBody, - ) -> Result> { - self.webdav_custom_request(BodyRequestType::Put, path, size, content_type, body) - .await - } - async fn webdav_propfind( &self, path: &str, @@ -466,8 +446,26 @@ impl WebdavBackend { content_type: Option<&str>, body: AsyncBody, ) -> Result> { - self.webdav_custom_request(BodyRequestType::Propfind, path, size, content_type, body) - .await + let p = build_abs_path(&self.root, path); + + let url = format!("{}/{}", self.endpoint, percent_encode_path(&p)); + let mut req = Request::builder() + .method("PROPFIND") + .uri(&url) + .header(AUTHORIZATION, &self.authorization); + + if let Some(size) = size { + req = req.header(CONTENT_LENGTH, size) + } + + if let Some(mime) = content_type { + req = req.header(CONTENT_TYPE, mime) + } + + // Set body + let req = req.body(body).map_err(new_request_build_error)?; + + self.client.send_async(req).await } async fn webdav_head(&self, path: &str) -> Result> { diff --git a/src/services/webdav/body_request_type.rs b/src/services/webdav/body_request_type.rs deleted file mode 100644 index 8ac1c0e5f3c1..000000000000 --- a/src/services/webdav/body_request_type.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[derive(Debug, Clone)] -pub enum BodyRequestType { - Put, - Propfind, -} - -impl ToString for BodyRequestType { - fn to_string(&self) -> String { - match self { - BodyRequestType::Put => "PUT".to_string(), - BodyRequestType::Propfind => "PROPFIND".to_string(), - } - } -} diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 20eef9756a00..1c878dde0e3e 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -13,24 +13,20 @@ // limitations under the License. use crate::Result; -use crate::{ - raw::{normalize_path, output}, - ObjectMetadata, ObjectMode, -}; +use crate::{raw::output, ObjectMetadata, ObjectMode}; use async_trait::async_trait; use super::list_response::Multistatus; -use std::path::{Path, PathBuf}; pub struct DirStream { - root: PathBuf, + root: String, size: usize, multistates: Multistatus, } impl DirStream { - pub fn new(root: &Path, multistates: Multistatus, limit: Option) -> Self { + pub fn new(root: String, multistates: Multistatus, limit: Option) -> Self { Self { root: root.to_owned(), size: limit.unwrap_or(1000), @@ -50,22 +46,14 @@ impl output::Page for DirStream { } match self.multistates.response.get(0) { Some(de) => { - let path = PathBuf::from(de.href.clone()); - - let rel_path = normalize_path( - &path - .strip_prefix(&self.root) - .expect("cannot fail because the prefix is iterated") - .to_string_lossy() - .replace('\\', "/"), - ); + let path = de.href.clone(); let entry = if de.propstat.prop.resourcetype.value == Some(super::list_response::ResourceType::Collection) { - output::Entry::new(&rel_path, ObjectMetadata::new(ObjectMode::DIR)) + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) } else { - output::Entry::new(&rel_path, ObjectMetadata::new(ObjectMode::FILE)) + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) }; oes.push(entry); diff --git a/src/services/webdav/mod.rs b/src/services/webdav/mod.rs index 0de33aefb3e0..90cc4694ea71 100644 --- a/src/services/webdav/mod.rs +++ b/src/services/webdav/mod.rs @@ -15,7 +15,6 @@ mod backend; pub use backend::WebdavBuilder as Webdav; -mod body_request_type; mod dir_stream; mod error; mod list_response; From 0a361f3e891be2c8cc85c2200da0858fdc57d384 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 22:14:37 -0800 Subject: [PATCH 17/56] refine lock & clean up --- src/services/webdav/backend.rs | 5 +- src/services/webdav/dir_stream.rs | 5 +- src/services/webdav/list_response.rs | 92 +--------------------------- 3 files changed, 3 insertions(+), 99 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index ab78b7e3a86f..34d0eca3b505 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -292,10 +292,7 @@ impl Accessor for WebdavBackend { .with_context("service", Scheme::Webdav) })?; - Ok(( - RpList::default(), - DirStream::new(self.root.clone(), result, args.limit()), - )) + Ok((RpList::default(), DirStream::new(result, args.limit()))) } _ => Err(parse_error(resp).await?), // TODO: handle error gracefully } diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 1c878dde0e3e..ccf0bb75618d 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -19,16 +19,13 @@ use async_trait::async_trait; use super::list_response::Multistatus; pub struct DirStream { - root: String, - size: usize, multistates: Multistatus, } impl DirStream { - pub fn new(root: String, multistates: Multistatus, limit: Option) -> Self { + pub fn new(multistates: Multistatus, limit: Option) -> Self { Self { - root: root.to_owned(), size: limit.unwrap_or(1000), multistates: multistates, } diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 5175fe551770..2fe4c284cbd6 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -14,13 +14,6 @@ use serde::Deserialize; -#[derive(Deserialize, Debug, PartialEq)] -#[serde(rename_all = "lowercase")] -pub struct LockEntry { - pub lockscope: LockScopeContainer, - pub locktype: LockTypeContainer, -} - #[derive(Deserialize, Debug, PartialEq)] pub struct LockScopeContainer { #[serde(rename = "$value")] @@ -68,7 +61,6 @@ pub struct Prop { pub getlastmodified: String, pub resourcetype: ResourceTypeContainer, pub lockdiscovery: (), - pub supportedlock: SupportedLock, } #[derive(Deserialize, Debug, PartialEq)] @@ -83,44 +75,10 @@ pub enum ResourceType { Collection, } -#[derive(Deserialize, Debug, PartialEq)] - -pub struct SupportedLock { - lockentry: LockEntry, -} - #[cfg(test)] mod tests { use super::*; use quick_xml::de::from_str; - #[test] - fn test_lockentry() { - let xml = r#" - - - "#; - - let lockentry = from_str::(xml).unwrap(); - assert_eq!(lockentry.lockscope.value, LockScope::Exclusive); - assert_eq!(lockentry.locktype.value, LockType::Write); - } - - #[test] - fn test_supportedlock() { - let xml = r#" - - - - - "#; - - let supportedlock = from_str::(xml).unwrap(); - assert_eq!( - supportedlock.lockentry.lockscope.value, - LockScope::Exclusive - ); - assert_eq!(supportedlock.lockentry.locktype.value, LockType::Write); - } #[test] fn test_propstat() { @@ -150,14 +108,7 @@ mod tests { propstat.prop.resourcetype.value.unwrap(), ResourceType::Collection ); - assert_eq!( - propstat.prop.supportedlock.lockentry.lockscope.value, - LockScope::Exclusive - ); - assert_eq!( - propstat.prop.supportedlock.lockentry.locktype.value, - LockType::Write - ); + assert_eq!(propstat.status, "HTTP/1.1 200 OK"); } @@ -193,26 +144,6 @@ mod tests { response.propstat.prop.resourcetype.value.unwrap(), ResourceType::Collection ); - assert_eq!( - response - .propstat - .prop - .supportedlock - .lockentry - .lockscope - .value, - LockScope::Exclusive - ); - assert_eq!( - response - .propstat - .prop - .supportedlock - .lockentry - .locktype - .value, - LockType::Write - ); assert_eq!(response.propstat.status, "HTTP/1.1 200 OK"); } @@ -250,27 +181,6 @@ mod tests { "Tue, 07 May 2022 05:52:22 GMT" ); assert_eq!(response.propstat.prop.resourcetype.value, None); - - assert_eq!( - response - .propstat - .prop - .supportedlock - .lockentry - .lockscope - .value, - LockScope::Exclusive - ); - assert_eq!( - response - .propstat - .prop - .supportedlock - .lockentry - .locktype - .value, - LockType::Write - ); assert_eq!(response.propstat.status, "HTTP/1.1 200 OK"); } From 3af0ddd2392c7d9bfea5cdc1f22001968ad7681b Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sun, 12 Feb 2023 22:16:03 -0800 Subject: [PATCH 18/56] fix logic --- src/services/webdav/dir_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index ccf0bb75618d..51593d11def1 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -41,7 +41,7 @@ impl output::Page for DirStream { if i >= self.multistates.response.len() { break; } - match self.multistates.response.get(0) { + match self.multistates.response.get(i) { Some(de) => { let path = de.href.clone(); From f458d12b54251cabdb09221e04e36147fc6965b9 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 14 Feb 2023 07:53:17 -0800 Subject: [PATCH 19/56] fix lint: dir_stream.rs --- src/services/webdav/dir_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 51593d11def1..9ba813792f0b 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -27,7 +27,7 @@ impl DirStream { pub fn new(multistates: Multistatus, limit: Option) -> Self { Self { size: limit.unwrap_or(1000), - multistates: multistates, + multistates, } } } From 9514fd050b64c481948493c53e44630668912135 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 14 Feb 2023 08:15:00 -0800 Subject: [PATCH 20/56] trying to fix --- src/services/webdav/dir_stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 9ba813792f0b..c3d0d020721e 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -35,7 +35,7 @@ impl DirStream { #[async_trait] impl output::Page for DirStream { async fn next_page(&mut self) -> Result>> { - let mut oes: Vec = Vec::with_capacity(self.size); + let mut oes: Vec = Vec::new(); for i in 0..self.size { if i >= self.multistates.response.len() { From 7e64e932bc960bcfe757c9a6b6bdfcb7d32fa4cb Mon Sep 17 00:00:00 2001 From: Daohan Chong Date: Tue, 14 Feb 2023 08:38:46 -0800 Subject: [PATCH 21/56] Update src/services/webdav/dir_stream.rs per comment from @ClSlaid Co-authored-by: ClSlaid --- src/services/webdav/dir_stream.rs | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index c3d0d020721e..861137a8bc90 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -37,26 +37,17 @@ impl output::Page for DirStream { async fn next_page(&mut self) -> Result>> { let mut oes: Vec = Vec::new(); - for i in 0..self.size { - if i >= self.multistates.response.len() { - break; - } - match self.multistates.response.get(i) { - Some(de) => { - let path = de.href.clone(); + while let Some(de) = self.multistates.response.pop() { + let path = de.href.clone(); - let entry = if de.propstat.prop.resourcetype.value + let entry = if de.propstat.prop.resourcetype.value == Some(super::list_response::ResourceType::Collection) - { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) - } else { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) - }; - - oes.push(entry); - } - None => break, - } + { + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) + } else { + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) + }; + oes.push(entry); } Ok(if oes.is_empty() { None } else { Some(oes) }) From aca39e6ca1202362c881eba177147b56ea423d10 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 14 Feb 2023 21:11:13 -0800 Subject: [PATCH 22/56] dav_ext_methods PROPFIND; --- src/services/webdav/fixtures/nginx.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/webdav/fixtures/nginx.conf b/src/services/webdav/fixtures/nginx.conf index f76ece6c1394..5882326da18f 100644 --- a/src/services/webdav/fixtures/nginx.conf +++ b/src/services/webdav/fixtures/nginx.conf @@ -16,6 +16,7 @@ http { client_body_temp_path /tmp; log_not_found off; dav_methods PUT DELETE; + dav_ext_methods PROPFIND; create_full_put_path on; client_max_body_size 1024M; } From ca2b122e163b0c3ea1442432844131bf3ca146a9 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 14 Feb 2023 21:11:37 -0800 Subject: [PATCH 23/56] PROPFIND --- src/services/webdav/fixtures/nginx-with-basic-auth.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/webdav/fixtures/nginx-with-basic-auth.conf b/src/services/webdav/fixtures/nginx-with-basic-auth.conf index 63d89768dc7e..3646228b315c 100644 --- a/src/services/webdav/fixtures/nginx-with-basic-auth.conf +++ b/src/services/webdav/fixtures/nginx-with-basic-auth.conf @@ -16,6 +16,7 @@ http { client_body_temp_path /tmp; log_not_found off; dav_methods PUT DELETE; + dav_ext_methods PROPFIND; create_full_put_path on; client_max_body_size 1024M; auth_basic "Administrator’s Area"; From ed5ee6e5e87f70866ca1cd60c0b36865fed2a66b Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 14 Feb 2023 22:06:20 -0800 Subject: [PATCH 24/56] fix formatting --- src/services/webdav/dir_stream.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 861137a8bc90..c502266a6efd 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -41,11 +41,11 @@ impl output::Page for DirStream { let path = de.href.clone(); let entry = if de.propstat.prop.resourcetype.value - == Some(super::list_response::ResourceType::Collection) + == Some(super::list_response::ResourceType::Collection) { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) } else { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) }; oes.push(entry); } From 8c9be54e860532c4ad04b92b199de4e9174eb9ff Mon Sep 17 00:00:00 2001 From: imWildCat Date: Tue, 14 Feb 2023 22:09:51 -0800 Subject: [PATCH 25/56] improve nginx config --- src/services/webdav/fixtures/nginx-with-basic-auth.conf | 2 ++ src/services/webdav/fixtures/nginx.conf | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/services/webdav/fixtures/nginx-with-basic-auth.conf b/src/services/webdav/fixtures/nginx-with-basic-auth.conf index 3646228b315c..1d3c2e926141 100644 --- a/src/services/webdav/fixtures/nginx-with-basic-auth.conf +++ b/src/services/webdav/fixtures/nginx-with-basic-auth.conf @@ -5,6 +5,8 @@ events { worker_connections 1024; } +include /etc/nginx/modules-enabled/*.conf; + http { server { listen 127.0.0.1:8080; diff --git a/src/services/webdav/fixtures/nginx.conf b/src/services/webdav/fixtures/nginx.conf index 5882326da18f..a7eac04aaf7f 100644 --- a/src/services/webdav/fixtures/nginx.conf +++ b/src/services/webdav/fixtures/nginx.conf @@ -5,6 +5,8 @@ events { worker_connections 1024; } +include /etc/nginx/modules-enabled/*.conf; + http { server { listen 127.0.0.1:8080; From 006f30d44426f2a6d8eb582afb0e065d632ad546 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 15 Feb 2023 14:52:35 +0800 Subject: [PATCH 26/56] Install nginx full Signed-off-by: Xuanwo --- .github/workflows/service_test_webdav.yml | 3 +++ src/services/webdav/fixtures/nginx-with-basic-auth.conf | 2 -- src/services/webdav/fixtures/nginx.conf | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/service_test_webdav.yml b/.github/workflows/service_test_webdav.yml index 080def285ac8..163196f8072f 100644 --- a/.github/workflows/service_test_webdav.yml +++ b/.github/workflows/service_test_webdav.yml @@ -24,6 +24,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Install nginx full for dav_ext modules + run: apt install nginx-full + - name: Start nginx shell: bash run: | diff --git a/src/services/webdav/fixtures/nginx-with-basic-auth.conf b/src/services/webdav/fixtures/nginx-with-basic-auth.conf index 1d3c2e926141..3646228b315c 100644 --- a/src/services/webdav/fixtures/nginx-with-basic-auth.conf +++ b/src/services/webdav/fixtures/nginx-with-basic-auth.conf @@ -5,8 +5,6 @@ events { worker_connections 1024; } -include /etc/nginx/modules-enabled/*.conf; - http { server { listen 127.0.0.1:8080; diff --git a/src/services/webdav/fixtures/nginx.conf b/src/services/webdav/fixtures/nginx.conf index a7eac04aaf7f..5882326da18f 100644 --- a/src/services/webdav/fixtures/nginx.conf +++ b/src/services/webdav/fixtures/nginx.conf @@ -5,8 +5,6 @@ events { worker_connections 1024; } -include /etc/nginx/modules-enabled/*.conf; - http { server { listen 127.0.0.1:8080; From e8af4477f44bfa5a0e9fcb77d1824a20e0fa94f0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 15 Feb 2023 15:08:36 +0800 Subject: [PATCH 27/56] Fix apt install Signed-off-by: Xuanwo --- .github/workflows/service_test_webdav.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/service_test_webdav.yml b/.github/workflows/service_test_webdav.yml index 163196f8072f..ed72f55c0382 100644 --- a/.github/workflows/service_test_webdav.yml +++ b/.github/workflows/service_test_webdav.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v3 - name: Install nginx full for dav_ext modules - run: apt install nginx-full + run: sudo apt install nginx-full - name: Start nginx shell: bash @@ -51,6 +51,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Install nginx full for dav_ext modules + run: sudo apt install nginx-full + - name: Start nginx shell: bash run: | From ca51c840a6a0462d3ca21d4c2d89c32dc3fb49bd Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 15 Feb 2023 15:38:23 +0800 Subject: [PATCH 28/56] import module Signed-off-by: Xuanwo --- src/services/webdav/fixtures/nginx-with-basic-auth.conf | 2 ++ src/services/webdav/fixtures/nginx.conf | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/services/webdav/fixtures/nginx-with-basic-auth.conf b/src/services/webdav/fixtures/nginx-with-basic-auth.conf index 3646228b315c..2d2367844ebf 100644 --- a/src/services/webdav/fixtures/nginx-with-basic-auth.conf +++ b/src/services/webdav/fixtures/nginx-with-basic-auth.conf @@ -1,3 +1,5 @@ +load_module /usr/lib/nginx/modules/ngx_http_dav_ext_module.so; + error_log /tmp/error.log; pid /tmp/nginx.pid; diff --git a/src/services/webdav/fixtures/nginx.conf b/src/services/webdav/fixtures/nginx.conf index 5882326da18f..7d9133ac3722 100644 --- a/src/services/webdav/fixtures/nginx.conf +++ b/src/services/webdav/fixtures/nginx.conf @@ -1,3 +1,5 @@ +load_module /usr/lib/nginx/modules/ngx_http_dav_ext_module.so; + error_log /tmp/error.log; pid /tmp/nginx.pid; From 08a0bf1d475180c90f6c167ff58279d0385e296d Mon Sep 17 00:00:00 2001 From: imWildCat Date: Wed, 15 Feb 2023 07:54:31 -0800 Subject: [PATCH 29/56] fix lint --- src/services/webdav/dir_stream.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index c502266a6efd..4b27d4f6fff2 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -36,18 +36,19 @@ impl DirStream { impl output::Page for DirStream { async fn next_page(&mut self) -> Result>> { let mut oes: Vec = Vec::new(); - - while let Some(de) = self.multistates.response.pop() { - let path = de.href.clone(); - - let entry = if de.propstat.prop.resourcetype.value - == Some(super::list_response::ResourceType::Collection) - { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) - } else { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) - }; - oes.push(entry); + for _ in 0..self.size { + if let Some(de) = self.multistates.response.pop() { + let path = de.href.clone(); + + let entry = if de.propstat.prop.resourcetype.value + == Some(super::list_response::ResourceType::Collection) + { + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) + } else { + output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) + }; + oes.push(entry); + } } Ok(if oes.is_empty() { None } else { Some(oes) }) From 3fbc28bbd21ed65b30421f06969cbc82a6709ca0 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 16 Feb 2023 19:50:14 -0800 Subject: [PATCH 30/56] nginx --- src/services/webdav/list_response.rs | 128 +++++++++++++++++++++++++-- 1 file changed, 119 insertions(+), 9 deletions(-) diff --git a/src/services/webdav/list_response.rs b/src/services/webdav/list_response.rs index 2fe4c284cbd6..07e3d6708d7b 100644 --- a/src/services/webdav/list_response.rs +++ b/src/services/webdav/list_response.rs @@ -57,10 +57,8 @@ pub struct Propstat { #[derive(Deserialize, Debug, PartialEq)] pub struct Prop { - pub displayname: String, pub getlastmodified: String, pub resourcetype: ResourceTypeContainer, - pub lockdiscovery: (), } #[derive(Deserialize, Debug, PartialEq)] @@ -99,7 +97,6 @@ mod tests { "#; let propstat = from_str::(xml).unwrap(); - assert_eq!(propstat.prop.displayname, "/"); assert_eq!( propstat.prop.getlastmodified, "Tue, 01 May 2022 06:39:47 GMT" @@ -135,7 +132,6 @@ mod tests { let response = from_str::(xml).unwrap(); assert_eq!(response.href, "/"); - assert_eq!(response.propstat.prop.displayname, "/"); assert_eq!( response.propstat.prop.getlastmodified, "Tue, 01 May 2022 06:39:47 GMT" @@ -175,7 +171,6 @@ mod tests { let response = from_str::(xml).unwrap(); assert_eq!(response.href, "/test_file"); - assert_eq!(response.propstat.prop.displayname, "test_file"); assert_eq!( response.propstat.prop.getlastmodified, "Tue, 07 May 2022 05:52:22 GMT" @@ -228,7 +223,6 @@ mod tests { let multistatus = from_str::(xml).unwrap(); assert_eq!(multistatus.response.len(), 2); assert_eq!(multistatus.response[0].href, "/"); - assert_eq!(multistatus.response[0].propstat.prop.displayname, "/"); assert_eq!( multistatus.response[0].propstat.prop.getlastmodified, "Tue, 01 May 2022 06:39:47 GMT" @@ -317,7 +311,6 @@ mod tests { assert_eq!(multistatus.response.len(), 3); let first_response = &multistatus.response[0]; assert_eq!(first_response.href, "/"); - assert_eq!(first_response.propstat.prop.displayname, "/"); assert_eq!( first_response.propstat.prop.getlastmodified, "Tue, 07 May 2022 06:39:47 GMT" @@ -325,7 +318,6 @@ mod tests { let second_response = &multistatus.response[1]; assert_eq!(second_response.href, "/testdir/"); - assert_eq!(second_response.propstat.prop.displayname, "testdir"); assert_eq!( second_response.propstat.prop.getlastmodified, "Tue, 07 May 2022 06:40:10 GMT" @@ -333,10 +325,128 @@ mod tests { let third_response = &multistatus.response[2]; assert_eq!(third_response.href, "/test_file"); - assert_eq!(third_response.propstat.prop.displayname, "test_file"); assert_eq!( third_response.propstat.prop.getlastmodified, "Tue, 07 May 2022 05:52:22 GMT" ); } + + #[test] + fn test_with_multiple_items_mixed_nginx() { + let xml = r#" + + + / + + + Fri, 17 Feb 2023 03:37:22 GMT + + + + + HTTP/1.1 200 OK + + + + /test_file_75 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_36 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_38 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_59 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_9 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_93 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_43 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + /test_file_95 + + + 1 + Fri, 17 Feb 2023 03:36:54 GMT + + + HTTP/1.1 200 OK + + + + "#; + + let multistatus: Multistatus = from_str(xml).unwrap(); + + assert_eq!(multistatus.response.len(), 9); + + let first_response = &multistatus.response[0]; + assert_eq!(first_response.href, "/"); + assert_eq!( + first_response.propstat.prop.getlastmodified, + "Fri, 17 Feb 2023 03:37:22 GMT" + ); + } } From 052baa42454c5db0b880a356d0f949606931e036 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 16 Feb 2023 19:56:07 -0800 Subject: [PATCH 31/56] build relative path --- src/services/webdav/backend.rs | 5 ++++- src/services/webdav/dir_stream.rs | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 34d0eca3b505..86411a7d89e8 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -292,7 +292,10 @@ impl Accessor for WebdavBackend { .with_context("service", Scheme::Webdav) })?; - Ok((RpList::default(), DirStream::new(result, args.limit()))) + Ok(( + RpList::default(), + DirStream::new(path, result, args.limit()), + )) } _ => Err(parse_error(resp).await?), // TODO: handle error gracefully } diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 4b27d4f6fff2..7bce39f114c2 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::raw::build_rel_path; use crate::Result; use crate::{raw::output, ObjectMetadata, ObjectMode}; use async_trait::async_trait; @@ -19,13 +20,15 @@ use async_trait::async_trait; use super::list_response::Multistatus; pub struct DirStream { + root: String, size: usize, multistates: Multistatus, } impl DirStream { - pub fn new(multistates: Multistatus, limit: Option) -> Self { + pub fn new(root: &str, multistates: Multistatus, limit: Option) -> Self { Self { + root: root.into(), size: limit.unwrap_or(1000), multistates, } @@ -39,13 +42,14 @@ impl output::Page for DirStream { for _ in 0..self.size { if let Some(de) = self.multistates.response.pop() { let path = de.href.clone(); + let normalized_path = &build_rel_path(&self.root, &path); let entry = if de.propstat.prop.resourcetype.value == Some(super::list_response::ResourceType::Collection) { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::DIR)) + output::Entry::new(normalized_path, ObjectMetadata::new(ObjectMode::DIR)) } else { - output::Entry::new(&path, ObjectMetadata::new(ObjectMode::FILE)) + output::Entry::new(normalized_path, ObjectMetadata::new(ObjectMode::FILE)) }; oes.push(entry); } From 1f7b64733a84db5538047f0e76bf7acfaaa9386a Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 16 Feb 2023 21:28:36 -0800 Subject: [PATCH 32/56] trying to fix self.root --- src/services/webdav/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 07482a081a98..a1d6fc27f79b 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -310,7 +310,7 @@ impl Accessor for WebdavBackend { Ok(( RpList::default(), - DirStream::new(path, result, args.limit()), + DirStream::new(&self.root, result, args.limit()), )) } _ => Err(parse_error(resp).await?), // TODO: handle error gracefully From 59c312315c4efe1f1e5f7f3b7b5e9eb066bafed3 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Thu, 16 Feb 2023 21:46:51 -0800 Subject: [PATCH 33/56] add default depth for propfind --- src/services/webdav/backend.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index a1d6fc27f79b..c40a9952f54b 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -483,7 +483,8 @@ impl WebdavBackend { let mut req = Request::builder() .method("PROPFIND") .uri(&url) - .header(AUTHORIZATION, &self.authorization); + .header(AUTHORIZATION, &self.authorization) + .header("Depth", "1"); if let Some(size) = size { req = req.header(CONTENT_LENGTH, size) From 09d89f84bb19e1fda373724980e49c0939275462 Mon Sep 17 00:00:00 2001 From: Young-Flash <71162630+Young-Flash@users.noreply.github.com> Date: Fri, 17 Feb 2023 18:26:20 +0800 Subject: [PATCH 34/56] feat: support auth for HttpBackend (#1359) support auth for HttpBackend --- src/services/http/backend.rs | 141 ++++++++++++++++++++++++++++++++- src/services/webdav/backend.rs | 2 +- 2 files changed, 140 insertions(+), 3 deletions(-) diff --git a/src/services/http/backend.rs b/src/services/http/backend.rs index 866f452fc58b..79a201f0a1b6 100644 --- a/src/services/http/backend.rs +++ b/src/services/http/backend.rs @@ -17,6 +17,9 @@ use std::fmt::Debug; use std::fmt::Formatter; use async_trait::async_trait; +use base64::engine::general_purpose; +use base64::Engine; +use http::header::AUTHORIZATION; use http::Request; use http::Response; use http::StatusCode; @@ -78,6 +81,9 @@ use crate::*; #[derive(Default)] pub struct HttpBuilder { endpoint: Option, + username: Option, + password: Option, + token: Option, root: Option, http_client: Option, } @@ -106,6 +112,36 @@ impl HttpBuilder { self } + /// set password for http backend + /// + /// default: no password + pub fn username(&mut self, username: &str) -> &mut Self { + if !username.is_empty() { + self.username = Some(username.to_owned()); + } + self + } + + /// set password for http backend + /// + /// default: no password + pub fn password(&mut self, password: &str) -> &mut Self { + if !password.is_empty() { + self.password = Some(password.to_owned()); + } + self + } + + /// set bearer token for http backend + /// + /// default: no access token + pub fn token(&mut self, token: &str) -> &mut Self { + if !token.is_empty() { + self.token = Some(token.to_owned()); + } + self + } + /// Set root path of http backend. pub fn root(&mut self, root: &str) -> &mut Self { self.root = if root.is_empty() { @@ -138,6 +174,9 @@ impl Builder for HttpBuilder { map.get("root").map(|v| builder.root(v)); map.get("endpoint").map(|v| builder.endpoint(v)); + map.get("username").map(|v| builder.username(v)); + map.get("password").map(|v| builder.password(v)); + map.get("token").map(|v| builder.token(v)); builder } @@ -167,9 +206,36 @@ impl Builder for HttpBuilder { })? }; + // authorization via `Basic` or `Bearer` + let auth = match (&self.username, &self.password, &self.token) { + (Some(username), Some(password), None) => { + format!( + "Basic {}", + general_purpose::STANDARD.encode(format!("{username}:{password}")) + ) + } + (Some(username), None, None) => { + format!( + "Basic {}", + general_purpose::STANDARD.encode(format!("{username}:")) + ) + } + (None, None, Some(token)) => { + format!("Bearer {token}") + } + (None, Some(_), _) => { + return Err( + Error::new(ErrorKind::BackendConfigInvalid, "missing username") + .with_context("service", Scheme::Http), + ) + } + _ => String::default(), + }; + debug!("backend build finished: {:?}", &self); Ok(HttpBackend { endpoint: endpoint.to_string(), + authorization: auth, root, client, }) @@ -180,6 +246,7 @@ impl Builder for HttpBuilder { #[derive(Clone)] pub struct HttpBackend { endpoint: String, + authorization: String, root: String, client: HttpClient, } @@ -253,7 +320,11 @@ impl HttpBackend { let url = format!("{}{}", self.endpoint, percent_encode_path(&p)); - let mut req = Request::get(&url); + let mut req = if self.authorization.is_empty() { + Request::get(&url) + } else { + Request::get(&url).header(AUTHORIZATION, &self.authorization) + }; if !range.is_full() { req = req.header(http::header::RANGE, range.to_header()); @@ -271,7 +342,11 @@ impl HttpBackend { let url = format!("{}{}", self.endpoint, percent_encode_path(&p)); - let req = Request::head(&url); + let req = if self.authorization.is_empty() { + Request::head(&url) + } else { + Request::head(&url).header(AUTHORIZATION, &self.authorization) + }; let req = req .body(AsyncBody::Empty) @@ -284,6 +359,8 @@ impl HttpBackend { #[cfg(test)] mod tests { use anyhow::Result; + use wiremock::matchers::basic_auth; + use wiremock::matchers::bearer_token; use wiremock::matchers::method; use wiremock::matchers::path; use wiremock::Mock; @@ -319,6 +396,66 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_read_via_basic_auth() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + + let (username, password) = ("your_username", "your_password"); + + let mock_server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/hello")) + .and(basic_auth(username, password)) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-length", "13") + .set_body_string("Hello, World!"), + ) + .mount(&mock_server) + .await; + + let mut builder = HttpBuilder::default(); + builder.endpoint(&mock_server.uri()); + builder.root("/"); + builder.username(username).password(password); + let op = Operator::create(builder)?.finish(); + + let bs = op.object("hello").read().await?; + + assert_eq!(bs, b"Hello, World!"); + Ok(()) + } + + #[tokio::test] + async fn test_read_via_bearer_auth() -> Result<()> { + let _ = env_logger::builder().is_test(true).try_init(); + + let token = "your_token"; + + let mock_server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/hello")) + .and(bearer_token(token)) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-length", "13") + .set_body_string("Hello, World!"), + ) + .mount(&mock_server) + .await; + + let mut builder = HttpBuilder::default(); + builder.endpoint(&mock_server.uri()); + builder.root("/"); + builder.token(token); + let op = Operator::create(builder)?.finish(); + + let bs = op.object("hello").read().await?; + + assert_eq!(bs, b"Hello, World!"); + Ok(()) + } + #[tokio::test] async fn test_stat() -> Result<()> { let _ = env_logger::builder().is_test(true).try_init(); diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index c40a9952f54b..e984647f8850 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -213,7 +213,7 @@ impl Builder for WebdavBuilder { })? }; - // base64 encode + // authorization via `Basic` or `Bearer` let auth = match (&self.username, &self.password, &self.token) { (Some(username), Some(password), None) => { format!( From e9e18f6e9fccd4112474ef1e7eed595d18c9a21b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 17 Feb 2023 18:40:28 +0800 Subject: [PATCH 35/56] feat: Add batch delete support (#1357) * Save work Signed-off-by: Xuanwo * Refactor batch operater Signed-off-by: Xuanwo * Add docs Signed-off-by: Xuanwo * Add behavior test Signed-off-by: Xuanwo * Format code Signed-off-by: Xuanwo * Fix test on fs Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- src/lib.rs | 2 +- src/operator.rs | 149 ++++++++++++++++++++++++++- src/ops.rs | 21 +++- src/raw/rps.rs | 5 + src/services/s3/backend.rs | 183 ++++++++++++++++++++++++++++++++- src/services/webdav/backend.rs | 7 +- tests/behavior/write.rs | 30 ++++++ 7 files changed, 386 insertions(+), 11 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index ad1964377e02..9cc54b111b64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,7 +115,7 @@ mod tests { fn assert_size() { assert_eq!(88, size_of::()); assert_eq!(16, size_of::()); - assert_eq!(16, size_of::()); + assert_eq!(112, size_of::()); assert_eq!(208, size_of::()); assert_eq!(48, size_of::()); assert_eq!(184, size_of::()); diff --git a/src/operator.rs b/src/operator.rs index 4fbd1256154f..d33f26bbc308 100644 --- a/src/operator.rs +++ b/src/operator.rs @@ -15,10 +15,13 @@ use std::collections::HashMap; use std::sync::Arc; +use futures::stream; +use futures::Stream; use futures::StreamExt; use futures::TryStreamExt; use crate::layers::*; +use crate::ops::*; use crate::raw::*; use crate::*; @@ -379,16 +382,127 @@ impl OperatorBuilder { #[derive(Clone, Debug)] pub struct BatchOperator { src: Operator, + meta: OperatorMetadata, + + limit: usize, } impl BatchOperator { pub(crate) fn new(op: Operator) -> Self { - BatchOperator { src: op } + let meta = op.metadata(); + + BatchOperator { + src: op, + meta, + limit: 1000, + } + } + + /// Specify the batch limit. + /// + /// Default: 1000 + pub fn with_limit(mut self, limit: usize) -> Self { + self.limit = limit; + self + } + + /// remove will given pathes. + /// + /// # Notes + /// + /// If underlying services support delete in batch, we will use batch + /// delete instead. + /// + /// # Examples + /// + /// ``` + /// # use anyhow::Result; + /// # use futures::io; + /// # use opendal::Operator; + /// # + /// # #[tokio::main] + /// # async fn test(op: Operator) -> Result<()> { + /// op.batch() + /// .remove(vec!["abc".to_string(), "def".to_string()]) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn remove(&self, pathes: Vec) -> Result<()> { + self.remove_via(stream::iter(pathes)).await + } + + /// remove_via will remove objects via given stream. + /// + /// We will delete by chunks with given batch limit on the stream. + /// + /// # Notes + /// + /// If underlying services support delete in batch, we will use batch + /// delete instead. + /// + /// # Examples + /// + /// ``` + /// # use anyhow::Result; + /// # use futures::io; + /// # use opendal::Operator; + /// use futures::stream; + /// # + /// # #[tokio::main] + /// # async fn test(op: Operator) -> Result<()> { + /// let stream = stream::iter(vec!["abc".to_string(), "def".to_string()]); + /// op.batch().remove_via(stream).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn remove_via(&self, mut input: impl Stream + Unpin) -> Result<()> { + if self.meta.can_batch() { + let mut input = input.map(|v| (v, OpDelete::default())).chunks(self.limit); + + while let Some(batches) = input.next().await { + let results = self + .src + .accessor + .batch(OpBatch::new(BatchOperations::Delete(batches))) + .await?; + + let BatchedResults::Delete(results) = results.into_results(); + + // TODO: return error here directly seems not a good idea? + for (_, result) in results { + let _ = result?; + } + } + } else { + while let Some(path) = input.next().await { + self.src.accessor.delete(&path, OpDelete::default()).await?; + } + } + + Ok(()) } /// Remove the path and all nested dirs and files recursively. /// - /// **Use this function in cautions to avoid unexpected data loss.** + /// # Notes + /// + /// If underlying services support delete in batch, we will use batch + /// delete instead. + /// + /// # Examples + /// + /// ``` + /// # use anyhow::Result; + /// # use futures::io; + /// # use opendal::Operator; + /// # + /// # #[tokio::main] + /// # async fn test(op: Operator) -> Result<()> { + /// op.batch().remove_all("path/to/dir").await?; + /// # Ok(()) + /// # } + /// ``` pub async fn remove_all(&self, path: &str) -> Result<()> { let parent = self.src.object(path); let meta = parent.metadata().await?; @@ -398,7 +512,36 @@ impl BatchOperator { } let obs = self.src.object(path).scan().await?; - obs.try_for_each(|v| async move { v.delete().await }).await + + if self.meta.can_batch() { + let mut obs = obs.try_chunks(self.limit); + + while let Some(batches) = obs.next().await { + let batches = batches + .map_err(|err| err.1)? + .into_iter() + .map(|v| (v.path().to_string(), OpDelete::default())) + .collect(); + + let results = self + .src + .accessor + .batch(OpBatch::new(BatchOperations::Delete(batches))) + .await?; + + let BatchedResults::Delete(results) = results.into_results(); + + // TODO: return error here directly seems not a good idea? + for (_, result) in results { + let _ = result?; + } + } + } else { + obs.try_for_each(|v| async move { v.delete().await }) + .await?; + } + + Ok(()) } } diff --git a/src/ops.rs b/src/ops.rs index 8d620a70f8ce..78656a6caf23 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -267,18 +267,23 @@ impl From for PresignOperation { /// Args for `batch` operation. pub struct OpBatch { - op: BatchOperations, + ops: BatchOperations, } impl OpBatch { + /// Create a new batch options. + pub fn new(ops: BatchOperations) -> Self { + Self { ops } + } + /// Get operation from op. pub fn operation(&self) -> &BatchOperations { - &self.op + &self.ops } /// Consume OpBatch into BatchOperation pub fn into_operation(self) -> BatchOperations { - self.op + self.ops } } @@ -290,6 +295,16 @@ pub enum BatchOperations { Delete(Vec<(String, OpDelete)>), } +impl BatchOperations { + /// Return the operation of this batch. + pub fn operation(&self) -> Operation { + use BatchOperations::*; + match self { + Delete(_) => Operation::Delete, + } + } +} + /// Args for `read` operation. #[derive(Debug, Clone, Default)] pub struct OpRead { diff --git a/src/raw/rps.rs b/src/raw/rps.rs index 7bfd57e4c27d..99b714dd0e06 100644 --- a/src/raw/rps.rs +++ b/src/raw/rps.rs @@ -193,6 +193,11 @@ pub struct RpBatch { } impl RpBatch { + /// Create a new RpBatch. + pub fn new(results: BatchedResults) -> Self { + Self { results } + } + /// Consume RpBatch to get the batched results. pub fn into_results(self) -> BatchedResults { self.results diff --git a/src/services/s3/backend.rs b/src/services/s3/backend.rs index 42964e8536c7..9acf4808179a 100644 --- a/src/services/s3/backend.rs +++ b/src/services/s3/backend.rs @@ -1120,7 +1120,7 @@ impl Accessor for S3Backend { am.set_scheme(Scheme::S3) .set_root(&self.root) .set_name(&self.bucket) - .set_capabilities(Read | Write | List | Scan | Presign | Multipart) + .set_capabilities(Read | Write | List | Scan | Presign | Multipart | Batch) .set_hints(ReadStreamable); am @@ -1257,6 +1257,54 @@ impl Accessor for S3Backend { ))) } + async fn batch(&self, args: OpBatch) -> Result { + let ops = args.into_operation(); + match ops { + BatchOperations::Delete(ops) => { + if ops.len() > 1000 { + return Err(Error::new( + ErrorKind::Unsupported, + "s3 services only allow delete up to 1000 keys at once", + ) + .with_context("length", ops.len().to_string())); + } + + let pathes = ops.into_iter().map(|(p, _)| p).collect(); + + let resp = self.s3_delete_objects(pathes).await?; + + let status = resp.status(); + + if let StatusCode::OK = status { + let bs = resp.into_body().bytes().await?; + + let result: DeleteObjectsResult = quick_xml::de::from_reader(bs.reader()) + .map_err(parse_xml_deserialize_error)?; + + let mut batched_result = + Vec::with_capacity(result.deleted.len() + result.error.len()); + for i in result.deleted { + let path = build_rel_path(&self.root, &i.key); + batched_result.push((path, Ok(RpDelete::default()))); + } + // TODO: we should handle those errors with code. + for i in result.error { + let path = build_rel_path(&self.root, &i.key); + + batched_result.push(( + path, + Err(Error::new(ErrorKind::Unexpected, &format!("{i:?}"))), + )); + } + + Ok(RpBatch::new(BatchedResults::Delete(batched_result))) + } else { + Err(parse_error(resp).await?) + } + } + } + } + async fn create_multipart( &self, path: &str, @@ -1642,6 +1690,45 @@ impl S3Backend { self.client.send_async(req).await } + + async fn s3_delete_objects(&self, pathes: Vec) -> Result> { + let url = format!("{}/?delete", self.endpoint); + + let req = Request::post(&url); + + let content = quick_xml::se::to_string(&DeleteObjectsRequest { + object: pathes + .into_iter() + .map(|path| DeleteObjectsRequestObject { + key: build_abs_path(&self.root, &path), + }) + .collect(), + }) + .map_err(parse_xml_deserialize_error)?; + + // Make sure content length has been set to avoid post with chunked encoding. + let req = req.header(CONTENT_LENGTH, content.len()); + // Set content-type to `application/xml` to avoid mixed with form post. + let req = req.header(CONTENT_TYPE, "application/xml"); + // Set content-md5 as required by API. + let req = req.header("CONTENT-MD5", { + use base64::engine::general_purpose; + use base64::Engine as _; + + let mut hasher = md5::Md5::new(); + hasher.update(content.as_bytes()); + + general_purpose::STANDARD.encode(hasher.finalize()) + }); + + let mut req = req + .body(AsyncBody::Bytes(Bytes::from(content))) + .map_err(new_request_build_error)?; + + self.signer.sign(&mut req).map_err(new_request_sign_error)?; + + self.client.send_async(req).await + } } /// Result of CreateMultipartUpload @@ -1695,6 +1782,41 @@ struct CompleteMultipartUploadRequestPart { etag: String, } +/// Request of DeleteObjects. +#[derive(Default, Debug, Serialize)] +#[serde(default, rename = "Delete", rename_all = "PascalCase")] +struct DeleteObjectsRequest { + object: Vec, +} + +#[derive(Default, Debug, Serialize)] +#[serde(rename_all = "PascalCase")] +struct DeleteObjectsRequestObject { + key: String, +} + +/// Result of DeleteObjects. +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename = "DeleteResult", rename_all = "PascalCase")] +struct DeleteObjectsResult { + deleted: Vec, + error: Vec, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +struct DeleteObjectsResultDeleted { + key: String, +} + +#[derive(Default, Debug, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +struct DeleteObjectsResultError { + code: String, + key: String, + message: String, +} + #[cfg(test)] mod tests { use backon::BlockingRetryable; @@ -1832,4 +1954,63 @@ mod tests { .replace('"', """) ) } + + /// This example is from https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html#API_DeleteObjects_Examples + #[test] + fn test_serialize_delete_objects_request() { + let req = DeleteObjectsRequest { + object: vec![ + DeleteObjectsRequestObject { + key: "sample1.txt".to_string(), + }, + DeleteObjectsRequestObject { + key: "sample2.txt".to_string(), + }, + ], + }; + + let actual = quick_xml::se::to_string(&req).expect("must succeed"); + + pretty_assertions::assert_eq!( + actual, + r#" + + sample1.txt + + + sample2.txt + + "# + // Cleanup space and new line + .replace([' ', '\n'], "") + ) + } + + /// This example is from https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html#API_DeleteObjects_Examples + #[test] + fn test_deserialize_delete_objects_result() { + let bs = Bytes::from( + r#" + + + sample1.txt + + + sample2.txt + AccessDenied + Access Denied + + "#, + ); + + let out: DeleteObjectsResult = + quick_xml::de::from_reader(bs.reader()).expect("must success"); + + assert_eq!(out.deleted.len(), 1); + assert_eq!(out.deleted[0].key, "sample1.txt"); + assert_eq!(out.error.len(), 1); + assert_eq!(out.error[0].key, "sample2.txt"); + assert_eq!(out.error[0].code, "AccessDenied"); + assert_eq!(out.error[0].message, "Access Denied"); + } } diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index e984647f8850..ca7f98b7d7ad 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -76,9 +76,10 @@ use crate::*; /// // create backend builder /// let mut builder = Webdav::default(); /// -/// builder.endpoint("127.0.0.1") -/// .username("xxx") -/// .password("xxx"); +/// builder +/// .endpoint("127.0.0.1") +/// .username("xxx") +/// .password("xxx"); /// /// let op: Operator = Operator::create(builder)?.finish(); /// let _obj: Object = op.object("test_file"); diff --git a/tests/behavior/write.rs b/tests/behavior/write.rs index 702577aa8991..ce4821a968be 100644 --- a/tests/behavior/write.rs +++ b/tests/behavior/write.rs @@ -100,6 +100,7 @@ macro_rules! behavior_write_tests { test_delete_empty_dir, test_delete_with_special_chars, test_delete_not_existing, + test_delete_stream, ); )* }; @@ -867,3 +868,32 @@ pub async fn test_delete_not_existing(op: Operator) -> Result<()> { Ok(()) } + +// Delete via stream. +pub async fn test_delete_stream(op: Operator) -> Result<()> { + let dir = uuid::Uuid::new_v4().to_string(); + op.object(&format!("{dir}/")) + .create() + .await + .expect("creat must succeed"); + + let expected: Vec<_> = (0..100).collect(); + for path in expected.iter() { + op.object(&format!("{dir}/{path}")).create().await?; + } + + op.batch() + .with_limit(30) + .remove_via(futures::stream::iter(expected.clone()).map(|v| format!("{dir}/{v}"))) + .await?; + + // Stat it again to check. + for path in expected.iter() { + assert!( + !op.object(&format!("{dir}/{path}")).is_exist().await?, + "{path} should be removed" + ) + } + + Ok(()) +} From 69b40d4c9b2161305654591257bce368b6cf9100 Mon Sep 17 00:00:00 2001 From: ClSlaid Date: Fri, 17 Feb 2023 18:51:25 +0800 Subject: [PATCH 36/56] docs: clarify about opendal user defined client (#1356) * docs: clarify opendal user defined client 1. no auto redirection 2. examples for disable it Signed-off-by: ClSlaid * fix: make document test compilable Signed-off-by: ClSlaid --------- Signed-off-by: ClSlaid --- src/raw/http_util/client.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/raw/http_util/client.rs b/src/raw/http_util/client.rs index 0f9978e228ad..f6c1e6575b63 100644 --- a/src/raw/http_util/client.rs +++ b/src/raw/http_util/client.rs @@ -108,6 +108,27 @@ impl HttpClient { /// /// And this API is an internal API, OpenDAL could change it while bumping /// minor versions. + /// + /// ## Reminders + /// ### no auto redirect + /// OpenDAL will handle all HTTP responses, including redirections. + /// Auto redirect may cause OpenDAL to fail. + /// + /// For reqwest client, please make sure your client's redirect policy is `Policy::none()`. + /// ```no_run + /// # use anyhow::Result; + /// # use reqwest::redirect::Policy; + /// # fn main() -> Result<()> { + /// let _client = reqwest::ClientBuilder::new().redirect(Policy::none()).build()?; + /// # Ok(()) + /// # } + /// ``` + /// For ureq client, please make sure your client's redirect count is `0`: + /// ```no_run + /// # fn main() { + /// let _client = ureq::AgentBuilder::new().redirects(0).build(); + /// # } + /// ``` pub fn with_client(async_client: reqwest::Client, sync_client: ureq::Agent) -> Self { Self { async_client, From 1b2589e007426405c8ae32c1e81ec3784aa4a2c2 Mon Sep 17 00:00:00 2001 From: ClSlaid Date: Fri, 17 Feb 2023 18:58:04 +0800 Subject: [PATCH 37/56] fix(webhdfs): should prepend http:// scheme (#1354) * fix(webhdfs): should prepend http:// scheme Signed-off-by: ClSlaid * refact: check scheme existence only on create Signed-off-by: ClSlaid * fmt: make msrv-clippy happy Signed-off-by: ClSlaid --------- Signed-off-by: ClSlaid --- src/services/webhdfs/backend.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/services/webhdfs/backend.rs b/src/services/webhdfs/backend.rs index daa3821a212b..4db071ca621e 100644 --- a/src/services/webhdfs/backend.rs +++ b/src/services/webhdfs/backend.rs @@ -213,10 +213,17 @@ impl Builder for WebhdfsBuilder { debug!("building backend: {:?}", self); let root = normalize_root(&self.root.take().unwrap_or_default()); - debug!("backend use root {}", root); + debug!("backend use root {root}"); + // check scheme let endpoint = match self.endpoint.take() { - Some(e) => e, + Some(endpoint) => { + if endpoint.starts_with("http") { + endpoint + } else { + format!("http://{endpoint}") + } + } None => WEBHDFS_DEFAULT_ENDPOINT.to_string(), }; debug!("backend use endpoint {}", endpoint); From 538fb1869d42c452bbcf3415eb4d616d07caac54 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 17 Feb 2023 20:56:57 +0800 Subject: [PATCH 38/56] ci: Pin time <= 0.3.17 until we decide to bump MSRV (#1361) * ci: Pin time <= 0.3.17 until we decide to bump MSRV Signed-off-by: Xuanwo * Fix build Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 92d7c92c8652..d0e28c963633 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,7 +127,9 @@ suppaftp = { version = "4.5", default-features = false, features = [ "async-secure", "async-rustls", ], optional = true } -time = { version = "0.3.10", features = ["serde"] } +# time 0.3.18 bump MSRV to 1.62, let's pin to 0.3.17 until we decide +# to bump ours +time = { version = ">=0.3.10, <=0.3.17", features = ["serde"] } tokio = { version = "1.20", features = ["fs"] } tracing = { version = "0.1", optional = true } trust-dns-resolver = { version = "0.22", optional = true } From 9da48325ed5ab6ad488016e85d0d0ba04c7a93c9 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 17 Feb 2023 22:40:32 +0800 Subject: [PATCH 39/56] ci: Only run service test on changing (#1363) Only run service test on changing Signed-off-by: Xuanwo --- .github/workflows/service_test_azblob.yml | 7 +++++-- .github/workflows/service_test_azdfs.yml | 7 +++++-- .github/workflows/service_test_fs.yml | 9 ++++++--- .github/workflows/service_test_ftp.yml | 7 +++++-- .github/workflows/service_test_gcs.yml | 7 +++++-- .github/workflows/service_test_ghac.yml | 7 +++++-- .github/workflows/service_test_hdfs.yml | 7 +++++-- .github/workflows/service_test_http.yml | 7 +++++-- .github/workflows/service_test_ipfs.yml | 7 +++++-- .github/workflows/service_test_ipmfs.yml | 7 +++++-- .github/workflows/service_test_memcached.yml | 7 +++++-- .github/workflows/service_test_memory.yml | 7 +++++-- .github/workflows/service_test_moka.yml | 7 +++++-- .github/workflows/service_test_obs.yml | 7 +++++-- .github/workflows/service_test_oss.yml | 7 +++++-- .github/workflows/service_test_redis.yml | 7 +++++-- .github/workflows/service_test_rocksdb.yml | 7 +++++-- .github/workflows/service_test_s3.yml | 7 +++++-- .github/workflows/service_test_sled.yml | 7 +++++-- .github/workflows/service_test_webdav.yml | 7 +++++-- .github/workflows/service_test_webhdfs.yml | 7 +++++-- 21 files changed, 106 insertions(+), 43 deletions(-) diff --git a/.github/workflows/service_test_azblob.yml b/.github/workflows/service_test_azblob.yml index 5a85bef557e2..3c607850d31b 100644 --- a/.github/workflows/service_test_azblob.yml +++ b/.github/workflows/service_test_azblob.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/azblob/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_azdfs.yml b/.github/workflows/service_test_azdfs.yml index a6459ba81712..aee53fe6f2dc 100644 --- a/.github/workflows/service_test_azdfs.yml +++ b/.github/workflows/service_test_azdfs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/azdfs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_fs.yml b/.github/workflows/service_test_fs.yml index 9c85358a8201..3be9b6215013 100644 --- a/.github/workflows/service_test_fs.yml +++ b/.github/workflows/service_test_fs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/fs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} @@ -21,7 +24,7 @@ jobs: matrix: os: - ubuntu-latest - - windows-latest + - windows-latest steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v2 diff --git a/.github/workflows/service_test_ftp.yml b/.github/workflows/service_test_ftp.yml index 6d9e2a376401..f9b619e75afd 100644 --- a/.github/workflows/service_test_ftp.yml +++ b/.github/workflows/service_test_ftp.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/ftp/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_gcs.yml b/.github/workflows/service_test_gcs.yml index ae7da73d251f..d8df6eceabb7 100644 --- a/.github/workflows/service_test_gcs.yml +++ b/.github/workflows/service_test_gcs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/gcs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_ghac.yml b/.github/workflows/service_test_ghac.yml index 15c007d70181..e2152511e656 100644 --- a/.github/workflows/service_test_ghac.yml +++ b/.github/workflows/service_test_ghac.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/ghac/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml index cc803922d560..208ef28c0461 100644 --- a/.github/workflows/service_test_hdfs.yml +++ b/.github/workflows/service_test_hdfs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/hdfs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_http.yml b/.github/workflows/service_test_http.yml index b1c41c62e507..7a689a1a9ccd 100644 --- a/.github/workflows/service_test_http.yml +++ b/.github/workflows/service_test_http.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/http/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_ipfs.yml b/.github/workflows/service_test_ipfs.yml index 8941e6d7139e..528aaa2d14d2 100644 --- a/.github/workflows/service_test_ipfs.yml +++ b/.github/workflows/service_test_ipfs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/ipfs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_ipmfs.yml b/.github/workflows/service_test_ipmfs.yml index 44f139f752ed..d9926758cb28 100644 --- a/.github/workflows/service_test_ipmfs.yml +++ b/.github/workflows/service_test_ipmfs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/ipmfs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_memcached.yml b/.github/workflows/service_test_memcached.yml index 918c13c88d61..be65a051109e 100644 --- a/.github/workflows/service_test_memcached.yml +++ b/.github/workflows/service_test_memcached.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/memcached/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_memory.yml b/.github/workflows/service_test_memory.yml index eddc7fef19f9..6c2b8755ee1b 100644 --- a/.github/workflows/service_test_memory.yml +++ b/.github/workflows/service_test_memory.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/memory/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_moka.yml b/.github/workflows/service_test_moka.yml index 168cb1bc1bbd..5e5c058cd09b 100644 --- a/.github/workflows/service_test_moka.yml +++ b/.github/workflows/service_test_moka.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/moka/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_obs.yml b/.github/workflows/service_test_obs.yml index d11552cf0c53..ca887189f10c 100644 --- a/.github/workflows/service_test_obs.yml +++ b/.github/workflows/service_test_obs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/obs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_oss.yml b/.github/workflows/service_test_oss.yml index 45c0390d0e15..9a8c072b5df9 100644 --- a/.github/workflows/service_test_oss.yml +++ b/.github/workflows/service_test_oss.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/oss/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index 818d232e41f9..0edd516ef7fc 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/redis/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_rocksdb.yml b/.github/workflows/service_test_rocksdb.yml index 0fb19ad1adad..30af2e0cf4ba 100644 --- a/.github/workflows/service_test_rocksdb.yml +++ b/.github/workflows/service_test_rocksdb.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/rocksdb/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 3de83bbbc9c3..b05c628d8938 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/s3/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_sled.yml b/.github/workflows/service_test_sled.yml index 31c2ffef5569..74281c21534e 100644 --- a/.github/workflows/service_test_sled.yml +++ b/.github/workflows/service_test_sled.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/sled/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_webdav.yml b/.github/workflows/service_test_webdav.yml index ed72f55c0382..bce1c69c7cee 100644 --- a/.github/workflows/service_test_webdav.yml +++ b/.github/workflows/service_test_webdav.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/webdav/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} diff --git a/.github/workflows/service_test_webhdfs.yml b/.github/workflows/service_test_webhdfs.yml index 977ad95e432b..7f50a1a4ccb2 100644 --- a/.github/workflows/service_test_webhdfs.yml +++ b/.github/workflows/service_test_webhdfs.yml @@ -7,8 +7,11 @@ on: pull_request: branches: - main - paths-ignore: - - "docs/**" + paths: + - "src/**" + - "!src/docs/**" + - "!src/services/**" + - "src/services/webhdfs/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} From 2abeb6aed932ea71c34dc553f9f2200d0dadb0f6 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 07:25:27 -0800 Subject: [PATCH 40/56] add auth for propfind --- src/services/webdav/backend.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index ca7f98b7d7ad..a16658d2664f 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -487,6 +487,12 @@ impl WebdavBackend { .header(AUTHORIZATION, &self.authorization) .header("Depth", "1"); + let mut req = if self.authorization.is_empty() { + Request::put(&url) + } else { + Request::put(&url).header(AUTHORIZATION, &self.authorization) + }; + if let Some(size) = size { req = req.header(CONTENT_LENGTH, size) } From cf9f215e4b0d45b985f173a9ba8609a737c87d94 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 09:01:12 -0800 Subject: [PATCH 41/56] fix list op --- src/services/webdav/backend.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index a16658d2664f..3cfb47e34d4f 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -487,10 +487,8 @@ impl WebdavBackend { .header(AUTHORIZATION, &self.authorization) .header("Depth", "1"); - let mut req = if self.authorization.is_empty() { - Request::put(&url) - } else { - Request::put(&url).header(AUTHORIZATION, &self.authorization) + if !self.authorization.is_empty() { + req = req.header(AUTHORIZATION, &self.authorization); }; if let Some(size) = size { From 1c621aff59fe447cbb75c8cd69861ef4681354ff Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 09:28:40 -0800 Subject: [PATCH 42/56] fix root path --- src/services/webdav/dir_stream.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 7bce39f114c2..267af2940d4a 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -42,7 +42,11 @@ impl output::Page for DirStream { for _ in 0..self.size { if let Some(de) = self.multistates.response.pop() { let path = de.href.clone(); - let normalized_path = &build_rel_path(&self.root, &path); + let normalized_path = &if self.root != path { + build_rel_path(&self.root, &path) + } else { + path + }; let entry = if de.propstat.prop.resourcetype.value == Some(super::list_response::ResourceType::Collection) From cd67d5d49d8eee898f40d2c320c87a7483c758b9 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 10:06:57 -0800 Subject: [PATCH 43/56] add xml header --- src/services/webdav/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 3cfb47e34d4f..1aed3cc912c9 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -289,6 +289,7 @@ impl Accessor for WebdavBackend { async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> { let all_prop_xml_body = r#" + @@ -484,7 +485,6 @@ impl WebdavBackend { let mut req = Request::builder() .method("PROPFIND") .uri(&url) - .header(AUTHORIZATION, &self.authorization) .header("Depth", "1"); if !self.authorization.is_empty() { From 9ed2094d301d80b65f2b51673f340c6c58efab49 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 10:20:59 -0800 Subject: [PATCH 44/56] fix prop xml --- src/services/webdav/backend.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 1aed3cc912c9..02727ef73def 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -288,8 +288,8 @@ impl Accessor for WebdavBackend { } async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> { - let all_prop_xml_body = r#" - + // XML body must start without a new line. Otherwise, the server will panic: `xmlParseChunk() failed` + let all_prop_xml_body = r#" From 6b42f6b5195d259b0f7cadb08b18c0ecf2945e9c Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 10:21:17 -0800 Subject: [PATCH 45/56] remove TODO --- src/services/webdav/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 02727ef73def..ba7ff676a1c9 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -315,7 +315,7 @@ impl Accessor for WebdavBackend { DirStream::new(&self.root, result, args.limit()), )) } - _ => Err(parse_error(resp).await?), // TODO: handle error gracefully + _ => Err(parse_error(resp).await?), } } From b04b300b4387b70c13d837a5ac3054bd592f2572 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 11:36:54 -0800 Subject: [PATCH 46/56] skip current path while listing --- src/services/webdav/backend.rs | 2 +- src/services/webdav/dir_stream.rs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index ba7ff676a1c9..03d00e1aeff0 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -312,7 +312,7 @@ impl Accessor for WebdavBackend { Ok(( RpList::default(), - DirStream::new(&self.root, result, args.limit()), + DirStream::new(&self.root, path, result, args.limit()), )) } _ => Err(parse_error(resp).await?), diff --git a/src/services/webdav/dir_stream.rs b/src/services/webdav/dir_stream.rs index 267af2940d4a..f13bb6d46653 100644 --- a/src/services/webdav/dir_stream.rs +++ b/src/services/webdav/dir_stream.rs @@ -21,14 +21,16 @@ use super::list_response::Multistatus; pub struct DirStream { root: String, + path: String, size: usize, multistates: Multistatus, } impl DirStream { - pub fn new(root: &str, multistates: Multistatus, limit: Option) -> Self { + pub fn new(root: &str, path: &str, multistates: Multistatus, limit: Option) -> Self { Self { root: root.into(), + path: path.into(), size: limit.unwrap_or(1000), multistates, } @@ -48,6 +50,11 @@ impl output::Page for DirStream { path }; + if normalized_path.eq(&self.path) { + // WebDav server may return the current path as an entry. + continue; + } + let entry = if de.propstat.prop.resourcetype.value == Some(super::list_response::ResourceType::Collection) { From a86f1a86d0ff7192dc2a337f81b7dd1a7a6c253f Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 20:53:10 -0800 Subject: [PATCH 47/56] handle 404 for dir --- src/services/webdav/backend.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 03d00e1aeff0..5c65e563f993 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -315,6 +315,17 @@ impl Accessor for WebdavBackend { DirStream::new(&self.root, path, result, args.limit()), )) } + StatusCode::NOT_FOUND if path.ends_with('/') => Ok(( + RpList::default(), + DirStream::new( + &self.root, + path, + Multistatus { + response: Vec::new(), + }, + args.limit(), + ), + )), _ => Err(parse_error(resp).await?), } } From 7f48a5c805189b05877f80aac60a7e4775553d0a Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 21:04:05 -0800 Subject: [PATCH 48/56] add MKCOL to fix mkdir --- src/services/webdav/fixtures/nginx-with-basic-auth.conf | 2 +- src/services/webdav/fixtures/nginx.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/webdav/fixtures/nginx-with-basic-auth.conf b/src/services/webdav/fixtures/nginx-with-basic-auth.conf index 2d2367844ebf..384defa3abdd 100644 --- a/src/services/webdav/fixtures/nginx-with-basic-auth.conf +++ b/src/services/webdav/fixtures/nginx-with-basic-auth.conf @@ -17,7 +17,7 @@ http { location / { client_body_temp_path /tmp; log_not_found off; - dav_methods PUT DELETE; + dav_methods PUT DELETE MKCOL; dav_ext_methods PROPFIND; create_full_put_path on; client_max_body_size 1024M; diff --git a/src/services/webdav/fixtures/nginx.conf b/src/services/webdav/fixtures/nginx.conf index 7d9133ac3722..22ab20df8b36 100644 --- a/src/services/webdav/fixtures/nginx.conf +++ b/src/services/webdav/fixtures/nginx.conf @@ -17,7 +17,7 @@ http { location / { client_body_temp_path /tmp; log_not_found off; - dav_methods PUT DELETE; + dav_methods PUT DELETE MKCOL; dav_ext_methods PROPFIND; create_full_put_path on; client_max_body_size 1024M; From 2e1e9666b01c30bcf0236968744efac021b92374 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Fri, 17 Feb 2023 21:14:29 -0800 Subject: [PATCH 49/56] add MKCOL for dirs --- src/services/webdav/backend.rs | 40 ++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 5c65e563f993..c1f17b374826 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -331,9 +331,13 @@ impl Accessor for WebdavBackend { } async fn create(&self, path: &str, _: OpCreate) -> Result { - let resp = self - .webdav_put(path, Some(0), None, None, AsyncBody::Empty) - .await?; + let resp = if path.ends_with("/") { + self.webdav_mkcol(path, None, None, AsyncBody::Empty) + .await? + } else { + self.webdav_put(path, Some(0), None, None, AsyncBody::Empty) + .await? + }; let status = resp.status(); @@ -483,6 +487,35 @@ impl WebdavBackend { self.client.send_async(req).await } + async fn webdav_mkcol( + &self, + path: &str, + content_type: Option<&str>, + content_disposition: Option<&str>, + body: AsyncBody, + ) -> Result> { + let p = build_abs_path(&self.root, path); + + let url = format!("{}/{}", self.endpoint, percent_encode_path(&p)); + + let mut req = Request::builder().method("MKCOL").uri(&url); + if !self.authorization.is_empty() { + req = req.header(AUTHORIZATION, &self.authorization); + } + + if let Some(mime) = content_type { + req = req.header(CONTENT_TYPE, mime) + } + + if let Some(cd) = content_disposition { + req = req.header(CONTENT_DISPOSITION, cd) + } + + let req = req.body(body).map_err(new_request_build_error)?; + + self.client.send_async(req).await + } + async fn webdav_propfind( &self, path: &str, @@ -510,7 +543,6 @@ impl WebdavBackend { req = req.header(CONTENT_TYPE, mime) } - // Set body let req = req.body(body).map_err(new_request_build_error)?; self.client.send_async(req).await From ab273262edca85d03fc0d7651e17182ad9d447fd Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 07:05:11 -0800 Subject: [PATCH 50/56] end --- src/services/webdav/backend.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 4a3a9fd1c0e4..065eb1e05d9b 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -312,7 +312,7 @@ impl Accessor for WebdavBackend { } async fn create(&self, path: &str, _: OpCreate) -> Result { - let resp = if path.ends_with("/") { + let resp = if path.ends_with('/') { self.webdav_mkcol(path, None, None, AsyncBody::Empty) .await? } else { From ee9d7b4a88d257e3c85508ba9916f60bdea3b7c6 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 12:20:28 -0800 Subject: [PATCH 51/56] create dir recursively --- src/services/webdav/backend.rs | 58 ++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 065eb1e05d9b..f1feb3d34278 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -312,28 +312,21 @@ impl Accessor for WebdavBackend { } async fn create(&self, path: &str, _: OpCreate) -> Result { - let resp = if path.ends_with('/') { - self.webdav_mkcol(path, None, None, AsyncBody::Empty) - .await? - } else { - self.webdav_put(path, Some(0), None, None, AsyncBody::Empty) - .await? - }; - - let status = resp.status(); - - match status { - StatusCode::CREATED - | StatusCode::OK - // create existing dir will return conflict - | StatusCode::CONFLICT - // create existing file will return no_content - | StatusCode::NO_CONTENT => { - resp.into_body().consume().await?; - Ok(RpCreate::default()) + // create dir recursively, split path by `/` and create each dir except the last one + let parts = path.split('/'); + let paths_without_last_part: Vec<&str> = parts.clone().take(parts.count() - 1).collect(); + + let mut sub_path = String::new(); + for sub_part in paths_without_last_part { + if sub_part.is_empty() { + continue; } - _ => Err(parse_error(resp).await?), + let sub_path_with_slash = sub_path.clone() + "/"; + sub_path.push_str(&sub_path_with_slash); + self.create_internal(&sub_path).await?; } + + self.create_internal(path).await } async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { @@ -564,4 +557,29 @@ impl WebdavBackend { self.client.send_async(req).await } + + async fn create_internal(&self, path: &str) -> Result { + let resp = if path.ends_with('/') { + self.webdav_mkcol(path, None, None, AsyncBody::Empty) + .await? + } else { + self.webdav_put(path, Some(0), None, None, AsyncBody::Empty) + .await? + }; + + let status = resp.status(); + + match status { + StatusCode::CREATED + | StatusCode::OK + // create existing dir will return conflict + | StatusCode::CONFLICT + // create existing file will return no_content + | StatusCode::NO_CONTENT => { + resp.into_body().consume().await?; + Ok(RpCreate::default()) + } + _ => Err(parse_error(resp).await?), + } + } } From ef69945a9dc8c86066b8ea31a6bf7b461a774d28 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 12:33:19 -0800 Subject: [PATCH 52/56] handle StatusCode::METHOD_NOT_ALLOWED --- src/services/webdav/backend.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index f1feb3d34278..1d48a6b10dd9 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -572,6 +572,8 @@ impl WebdavBackend { match status { StatusCode::CREATED | StatusCode::OK + // `File exists` will return `Method Not Allowed` + | StatusCode::METHOD_NOT_ALLOWED // create existing dir will return conflict | StatusCode::CONFLICT // create existing file will return no_content From bf97bcceeba44d927c8e3ba38ab12e4bbe2d4398 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 12:40:26 -0800 Subject: [PATCH 53/56] fix iteration --- src/services/webdav/backend.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 1d48a6b10dd9..6b4fa2e3e097 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -313,11 +313,13 @@ impl Accessor for WebdavBackend { async fn create(&self, path: &str, _: OpCreate) -> Result { // create dir recursively, split path by `/` and create each dir except the last one - let parts = path.split('/'); - let paths_without_last_part: Vec<&str> = parts.clone().take(parts.count() - 1).collect(); + let mut parts: Vec<&str> = path.split('/').filter(|x| !x.is_empty()).collect(); + if !parts.is_empty() { + parts.pop(); + } let mut sub_path = String::new(); - for sub_part in paths_without_last_part { + for sub_part in parts { if sub_part.is_empty() { continue; } From aa4d2b4bab21dff1574103d1130b83543a534587 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 12:44:05 -0800 Subject: [PATCH 54/56] fix typo --- src/services/webdav/backend.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index 6b4fa2e3e097..dda3205f8fa9 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -320,10 +320,7 @@ impl Accessor for WebdavBackend { let mut sub_path = String::new(); for sub_part in parts { - if sub_part.is_empty() { - continue; - } - let sub_path_with_slash = sub_path.clone() + "/"; + let sub_path_with_slash = sub_part.clone().to_owned() + "/"; sub_path.push_str(&sub_path_with_slash); self.create_internal(&sub_path).await?; } From 63acd2db56f06f2a0c5e70623926ae6db69978de Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 13:21:07 -0800 Subject: [PATCH 55/56] fix dir creation --- src/services/webdav/backend.rs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index dda3205f8fa9..f5dc371487dc 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -313,19 +313,21 @@ impl Accessor for WebdavBackend { async fn create(&self, path: &str, _: OpCreate) -> Result { // create dir recursively, split path by `/` and create each dir except the last one - let mut parts: Vec<&str> = path.split('/').filter(|x| !x.is_empty()).collect(); + let abs_path = build_abs_path(&self.root, path); + let abs_path = abs_path.as_str(); + let mut parts: Vec<&str> = abs_path.split('/').filter(|x| !x.is_empty()).collect(); if !parts.is_empty() { parts.pop(); } let mut sub_path = String::new(); for sub_part in parts { - let sub_path_with_slash = sub_part.clone().to_owned() + "/"; + let sub_path_with_slash = sub_part.to_owned() + "/"; sub_path.push_str(&sub_path_with_slash); self.create_internal(&sub_path).await?; } - self.create_internal(path).await + self.create_internal(abs_path).await } async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { @@ -426,15 +428,13 @@ impl WebdavBackend { async fn webdav_put( &self, - path: &str, + abs_path: &str, size: Option, content_type: Option<&str>, content_disposition: Option<&str>, body: AsyncBody, ) -> Result> { - let p = build_abs_path(&self.root, path); - - let url = format!("{}/{}", self.endpoint, percent_encode_path(&p)); + let url = format!("{}/{}", self.endpoint, percent_encode_path(abs_path)); let mut req = Request::put(&url); @@ -462,14 +462,12 @@ impl WebdavBackend { async fn webdav_mkcol( &self, - path: &str, + abs_path: &str, content_type: Option<&str>, content_disposition: Option<&str>, body: AsyncBody, ) -> Result> { - let p = build_abs_path(&self.root, path); - - let url = format!("{}/{}", self.endpoint, percent_encode_path(&p)); + let url = format!("{}/{}", self.endpoint, percent_encode_path(abs_path)); let mut req = Request::builder().method("MKCOL").uri(&url); if let Some(auth) = &self.authorization { @@ -557,12 +555,12 @@ impl WebdavBackend { self.client.send_async(req).await } - async fn create_internal(&self, path: &str) -> Result { - let resp = if path.ends_with('/') { - self.webdav_mkcol(path, None, None, AsyncBody::Empty) + async fn create_internal(&self, abs_path: &str) -> Result { + let resp = if abs_path.ends_with('/') { + self.webdav_mkcol(abs_path, None, None, AsyncBody::Empty) .await? } else { - self.webdav_put(path, Some(0), None, None, AsyncBody::Empty) + self.webdav_put(abs_path, Some(0), None, None, AsyncBody::Empty) .await? }; From 56a88ead9b2460a8b5ebb820f4433d81b6c3ffb8 Mon Sep 17 00:00:00 2001 From: imWildCat Date: Sat, 18 Feb 2023 13:29:01 -0800 Subject: [PATCH 56/56] fix write --- src/services/webdav/backend.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/webdav/backend.rs b/src/services/webdav/backend.rs index f5dc371487dc..2c6d6d34ef4e 100644 --- a/src/services/webdav/backend.rs +++ b/src/services/webdav/backend.rs @@ -345,9 +345,10 @@ impl Accessor for WebdavBackend { } async fn write(&self, path: &str, args: OpWrite, r: input::Reader) -> Result { + let abs_path = &build_abs_path(&self.root, path); let resp = self .webdav_put( - path, + abs_path, Some(args.size()), args.content_type(), args.content_disposition(),