From ebbc718d41ded20ab8176efea388c9796da795ca Mon Sep 17 00:00:00 2001
From: Raphael Taylor-Davies <r.taylordavies@googlemail.com>
Date: Fri, 1 Sep 2023 11:28:03 +0100
Subject: [PATCH] Fix HttpStore

---
 object_store/src/http/client.rs | 51 +++++++++++++++++++--------------
 1 file changed, 29 insertions(+), 22 deletions(-)

diff --git a/object_store/src/http/client.rs b/object_store/src/http/client.rs
index 1d3df34db9d1..bc0176f3cfed 100644
--- a/object_store/src/http/client.rs
+++ b/object_store/src/http/client.rs
@@ -256,31 +256,38 @@ impl Client {
     }
 
     pub async fn copy(&self, from: &Path, to: &Path, overwrite: bool) -> Result<()> {
-        let from = self.path_url(from);
-        let to = self.path_url(to);
-        let method = Method::from_bytes(b"COPY").unwrap();
-
-        let mut builder = self
-            .client
-            .request(method, from)
-            .header("Destination", to.as_str());
+        let mut retry = false;
+        loop {
+            let method = Method::from_bytes(b"COPY").unwrap();
 
-        if !overwrite {
-            builder = builder.header("Overwrite", "F");
-        }
+            let mut builder = self
+                .client
+                .request(method, self.path_url(from))
+                .header("Destination", self.path_url(to).as_str());
 
-        match builder.send_retry(&self.retry_config).await {
-            Ok(_) => Ok(()),
-            Err(e)
-                if !overwrite
-                    && matches!(e.status(), Some(StatusCode::PRECONDITION_FAILED)) =>
-            {
-                Err(crate::Error::AlreadyExists {
-                    path: to.to_string(),
-                    source: Box::new(e),
-                })
+            if !overwrite {
+                builder = builder.header("Overwrite", "F");
             }
-            Err(source) => Err(Error::Request { source }.into()),
+
+            return match builder.send_retry(&self.retry_config).await {
+                Ok(_) => Ok(()),
+                Err(source) => Err(match source.status() {
+                    Some(StatusCode::PRECONDITION_FAILED) if !overwrite => {
+                        crate::Error::AlreadyExists {
+                            path: to.to_string(),
+                            source: Box::new(source),
+                        }
+                        .into()
+                    }
+                    // Some implementations return 404 instead of 409
+                    Some(StatusCode::CONFLICT | StatusCode::NOT_FOUND) if !retry => {
+                        retry = true;
+                        self.create_parent_directories(&to).await?;
+                        continue;
+                    }
+                    _ => Error::Request { source }.into(),
+                }),
+            };
         }
     }
 }