diff --git a/core/Cargo.lock b/core/Cargo.lock index c593abffd48..b65f068c5aa 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -1590,6 +1590,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap 2.7.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.95", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.2.7" @@ -2098,6 +2117,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -5145,6 +5174,7 @@ dependencies = [ "chrono", "compio", "crc32c", + "crc64fast-nvme", "criterion", "dashmap 6.1.0", "dotenvy", @@ -7364,6 +7394,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -8580,11 +8619,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -8593,6 +8647,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap 2.7.0", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] diff --git a/core/Cargo.toml b/core/Cargo.toml index 600b5130393..d67ad7738e2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -192,6 +192,7 @@ services-s3 = [ "reqsign?/services-aws", "reqsign?/reqwest_request", "dep:crc32c", + "dep:crc64fast-nvme", ] services-seafile = [] services-sftp = ["dep:openssh", "dep:openssh-sftp-client", "dep:bb8"] @@ -352,6 +353,7 @@ compio = { version = "0.12.0", optional = true, features = [ ] } # for services-s3 crc32c = { version = "0.6.6", optional = true } +crc64fast-nvme = {version = "1.1.1", optional = true} # for services-nebula-graph rust-nebula = { version = "^0.0.2", optional = true, features = ["graph"] } snowflaked = { version = "1", optional = true, features = ["sync"] } diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 9145f22a7c8..2875a616750 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -750,6 +750,7 @@ impl Builder for S3Builder { let checksum_algorithm = match self.config.checksum_algorithm.as_deref() { Some("crc32c") => Some(ChecksumAlgorithm::Crc32c), + Some("crc64nvme") => Some(ChecksumAlgorithm::Crc64nvme), None => None, v => { return Err(Error::new( diff --git a/core/src/services/s3/config.rs b/core/src/services/s3/config.rs index 0c40d73f85f..60717090ec0 100644 --- a/core/src/services/s3/config.rs +++ b/core/src/services/s3/config.rs @@ -179,6 +179,7 @@ pub struct S3Config { /// /// Available options: /// - "crc32c" + /// - "crc64nvme" pub checksum_algorithm: Option, /// Disable write with if match so that opendal will not send write request with if match headers. /// diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs index 69cab8f2b93..5235d15a537 100644 --- a/core/src/services/s3/core.rs +++ b/core/src/services/s3/core.rs @@ -266,6 +266,12 @@ impl S3Core { body.clone() .for_each(|b| crc = crc32c::crc32c_append(crc, &b)); Some(BASE64_STANDARD.encode(crc.to_be_bytes())) + }, + Some(ChecksumAlgorithm::Crc64nvme) => { + let mut c = crc64fast_nvme::Digest::new(); + body.clone() + .for_each(|b| c.write(&b)); + Some(BASE64_STANDARD.encode(c.sum64().to_be_bytes())) } } } @@ -938,6 +944,8 @@ pub struct CompleteMultipartUploadRequestPart { pub etag: String, #[serde(rename = "ChecksumCRC32C", skip_serializing_if = "Option::is_none")] pub checksum_crc32c: Option, + #[serde(rename = "ChecksumCRC64NVME", skip_serializing_if = "Option::is_none")] + pub checksum_crc64nvme: Option, } /// Request of DeleteObjects. @@ -1046,11 +1054,13 @@ pub struct ListObjectVersionsOutputDeleteMarker { pub enum ChecksumAlgorithm { Crc32c, + Crc64nvme, } impl ChecksumAlgorithm { pub fn to_header_name(&self) -> HeaderName { match self { Self::Crc32c => HeaderName::from_static("x-amz-checksum-crc32c"), + Self::Crc64nvme => HeaderName::from_static("x-amz-checksum-crc64nvme"), } } } @@ -1061,6 +1071,7 @@ impl Display for ChecksumAlgorithm { "{}", match self { Self::Crc32c => "CRC32C", + Self::Crc64nvme => "CRC64NVME", } ) } diff --git a/core/src/services/s3/writer.rs b/core/src/services/s3/writer.rs index e0c7c5084b5..9b7beb3aa8d 100644 --- a/core/src/services/s3/writer.rs +++ b/core/src/services/s3/writer.rs @@ -147,6 +147,13 @@ impl oio::MultipartWrite for S3Writer { part_number: p.part_number, etag: p.etag.clone(), checksum_crc32c: p.checksum.clone(), + ..Default::default() + }, + ChecksumAlgorithm::Crc64nvme => CompleteMultipartUploadRequestPart { + part_number: p.part_number, + etag: p.etag.clone(), + checksum_crc64nvme: p.checksum.clone(), + ..Default::default() }, }, })