Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

aws s3 server_side_encryption configuration when upload object to s3 #5400

Open
wants to merge 23 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
78ec724
add aws s3 sse
samoii Sep 5, 2024
183de31
Merge branch 'main' of github.com:samoii/quickwit into aws-s3-sse
samoii Sep 7, 2024
716d39b
Merge branch 'main' of github.com:samoii/quickwit into aws-s3-sse
samoii Sep 10, 2024
f28677f
Merge branch 'main' of github.com:samoii/quickwit into aws-s3-sse
samoii Sep 12, 2024
42045be
server_side_encryption use enum varialble
samoii Sep 12, 2024
38af88b
remove unused
samoii Sep 12, 2024
4aad3c9
Merge pull request #1 from samoii/fix-s3-sse
samoii Sep 12, 2024
b8e5b78
add kms key id variable
samoii Sep 13, 2024
4e39153
Merge branch 'main' of github.com:samoii/quickwit into fix-s3-sse
samoii Sep 14, 2024
6319147
add kms key id
samoii Sep 14, 2024
21d8add
Merge pull request #2 from samoii/fix-s3-sse
samoii Sep 14, 2024
e6f0c16
function apply sse
samoii Sep 14, 2024
5147505
add sse to multipart upload and edit shown log
samoii Sep 14, 2024
ad68a93
Merge branch 'main' of github.com:samoii/quickwit into fix-s3-sse
samoii Sep 20, 2024
2996aee
fix lint
samoii Sep 20, 2024
c5d50d2
fix lint
samoii Sep 20, 2024
202c6e9
fix lint
samoii Sep 20, 2024
5730b1e
fix clippy
samoii Sep 21, 2024
4243a5f
fix rustfmt
samoii Sep 21, 2024
d9c4b8b
Merge pull request #3 from samoii/fix-s3-sse
samoii Sep 21, 2024
6ef96ad
Merge branch 'main' of github.com:samoii/quickwit into aws-s3-sse
samoii Sep 29, 2024
6529325
Merge branch 'main' of github.com:samoii/quickwit into aws-s3-sse
samoii Oct 3, 2024
660db5a
Merge branch 'main' of github.com:samoii/quickwit into aws-s3-sse
samoii Oct 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
function apply sse
  • Loading branch information
samoii committed Sep 14, 2024
commit e6f0c1685516f172c125ac23dabce0c4eb302cf8
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,25 @@ impl S3CompatibleObjectStorage {
.to_path_buf()
}

fn apply_server_side_encryption<'a>(
&'a self,
encryption: Option<S3ServerSideEncryption>,
kms_key_id: Option<String>,
) -> (Option<ServerSideEncryption>, Option<String>) {
let server_side_encryption = match encryption {
Some(S3ServerSideEncryption::Aes256) => Some(ServerSideEncryption::Aes256),
Some(S3ServerSideEncryption::AwsKms) => Some(ServerSideEncryption::AwsKms),
Some(S3ServerSideEncryption::AwsKmsDsse) => Some(ServerSideEncryption::AwsKmsDsse),
None => None,
};
let kms_key_id = match server_side_encryption {
Some(ServerSideEncryption::AwsKms) | Some(ServerSideEncryption::AwsKmsDsse) => kms_key_id,
_ => None,
};
(server_side_encryption, kms_key_id)
}


async fn put_single_part_single_try<'a>(
&'a self,
bucket: &'a str,
Expand All @@ -303,28 +322,37 @@ impl S3CompatibleObjectStorage {
.key(key)
.body(body)
.content_length(len as i64);
if let Some(encryption) = &self.server_side_encryption {
put_object_request = match encryption {
S3ServerSideEncryption::Aes256 => put_object_request.server_side_encryption(ServerSideEncryption::Aes256),
S3ServerSideEncryption::AwsKms => {
if let Some(kms_key_id) = &self.sse_kms_key_id {
put_object_request
.server_side_encryption(ServerSideEncryption::AwsKms)
.ssekms_key_id(kms_key_id)
} else {
put_object_request.server_side_encryption(ServerSideEncryption::AwsKms)
}
},
S3ServerSideEncryption::AwsKmsDsse => {
if let Some(kms_key_id) = &self.sse_kms_key_id {
put_object_request
.server_side_encryption(ServerSideEncryption::AwsKmsDsse)
.ssekms_key_id(kms_key_id)
} else {
put_object_request.server_side_encryption(ServerSideEncryption::AwsKmsDsse)
}
}
let (server_side_encryption, kms_key_id) = self.apply_server_side_encryption(
self.server_side_encryption.clone(),
self.sse_kms_key_id.clone()
);
if let Some(encryption) = server_side_encryption {
put_object_request = put_object_request.server_side_encryption(encryption);

if let Some(kms_key_id) = kms_key_id {
put_object_request = put_object_request.ssekms_key_id(kms_key_id);
}
// put_object_request = match encryption {
// S3ServerSideEncryption::Aes256 => put_object_request.server_side_encryption(ServerSideEncryption::Aes256),
// S3ServerSideEncryption::AwsKms => {
// if let Some(kms_key_id) = &self.sse_kms_key_id {
// put_object_request
// .server_side_encryption(ServerSideEncryption::AwsKms)
// .ssekms_key_id(kms_key_id)
// } else {
// put_object_request.server_side_encryption(ServerSideEncryption::AwsKms)
// }
// },
// S3ServerSideEncryption::AwsKmsDsse => {
// if let Some(kms_key_id) = &self.sse_kms_key_id {
// put_object_request
// .server_side_encryption(ServerSideEncryption::AwsKmsDsse)
// .ssekms_key_id(kms_key_id)
// } else {
// put_object_request.server_side_encryption(ServerSideEncryption::AwsKmsDsse)
// }
// }
// }
}
put_object_request
.send()
Expand Down