Skip to content

Commit

Permalink
s3proxy: handle server side encryption (sse)
Browse files Browse the repository at this point in the history 
Also handle unsigned payloads.
And correctly return ETags.
  • Loading branch information
@derpsteb
derpsteb committed Oct 4, 2023
1 parent f79824b commit bf24452
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 25 deletions.
48 changes: 39 additions & 9 deletions s3proxy/internal/router/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ type object struct {
objectLockLegalHoldStatus string
objectLockMode string
objectLockRetainUntilDate time.Time
sseCustomerAlgorithm string
sseCustomerKey string
sseCustomerKeyMD5 string
log *logger.Logger
}

Expand All @@ -54,7 +57,7 @@ func (o object) get(w http.ResponseWriter, r *http.Request) {
versionID = []string{""}
}

data, err := o.client.GetObject(r.Context(), o.bucket, o.key, versionID[0])
output, err := o.client.GetObject(r.Context(), o.bucket, o.key, versionID[0], o.sseCustomerAlgorithm, o.sseCustomerKey, o.sseCustomerKeyMD5)
if err != nil {
// log with Info as it might be expected behavior (e.g. object not found).
o.log.Errorf("GetObject sending request to S3", "error", err)
Expand All @@ -70,19 +73,46 @@ func (o object) get(w http.ResponseWriter, r *http.Request) {
return
}

if data.ETag != nil {
w.Header().Set("ETag", strings.Trim(*data.ETag, "\""))
if output.ETag != nil {
w.Header().Set("ETag", strings.Trim(*output.ETag, "\""))
}
if output.Expiration != nil {
w.Header().Set("x-amz-expiration", *output.Expiration)
}
if output.ChecksumCRC32 != nil {
w.Header().Set("x-amz-checksum-crc32", *output.ChecksumCRC32)
}
if output.ChecksumCRC32C != nil {
w.Header().Set("x-amz-checksum-crc32c", *output.ChecksumCRC32C)
}
if output.ChecksumSHA1 != nil {
w.Header().Set("x-amz-checksum-sha1", *output.ChecksumSHA1)
}
if output.ChecksumSHA256 != nil {
w.Header().Set("x-amz-checksum-sha256", *output.ChecksumSHA256)
}
if output.SSECustomerAlgorithm != nil {
w.Header().Set("x-amz-server-side-encryption-customer-algorithm", *output.SSECustomerAlgorithm)
}
if output.SSECustomerKeyMD5 != nil {
w.Header().Set("x-amz-server-side-encryption-customer-key-MD5", *output.SSECustomerKeyMD5)
}
if output.SSEKMSKeyId != nil {
w.Header().Set("x-amz-server-side-encryption-aws-kms-key-id", *output.SSEKMSKeyId)
}
if output.ServerSideEncryption != "" {
w.Header().Set("x-amz-server-side-encryption-context", string(output.ServerSideEncryption))
}

body, err := io.ReadAll(data.Body)
body, err := io.ReadAll(output.Body)
if err != nil {
o.log.Errorf("GetObject reading S3 response", "error", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}

plaintext := body
decrypt, ok := data.Metadata[encryptionTag]
decrypt, ok := output.Metadata[encryptionTag]

if ok && decrypt == "true" {
plaintext, err = crypto.Decrypt(body, []byte(testingKey))
Expand Down Expand Up @@ -111,7 +141,7 @@ func (o object) put(w http.ResponseWriter, r *http.Request) {
// GetObject needs to be able to recognize these objects and skip decryption.
o.metadata[encryptionTag] = "true"

output, err := o.client.PutObject(r.Context(), o.bucket, o.key, o.tags, o.contentType, o.objectLockLegalHoldStatus, o.objectLockMode, o.objectLockRetainUntilDate, o.metadata, ciphertext)
output, err := o.client.PutObject(r.Context(), o.bucket, o.key, o.tags, o.contentType, o.objectLockLegalHoldStatus, o.objectLockMode, o.sseCustomerAlgorithm, o.sseCustomerKey, o.sseCustomerKeyMD5, o.objectLockRetainUntilDate, o.metadata, ciphertext)
if err != nil {
o.log.Errorf("PutObject sending request to S3", "error", err)

Expand All @@ -132,7 +162,7 @@ func (o object) put(w http.ResponseWriter, r *http.Request) {
w.Header().Set("x-amz-version-id", *output.VersionId)
}
if output.ETag != nil {
w.Header().Set("ETag", *output.ETag)
w.Header().Set("ETag", strings.Trim(*output.ETag, "\""))
}
if output.Expiration != nil {
w.Header().Set("x-amz-expiration", *output.Expiration)
Expand Down Expand Up @@ -180,6 +210,6 @@ func parseErrorCode(err error) int {
}

type s3Client interface {
GetObject(ctx context.Context, bucket, key, versionID string) (*s3.GetObjectOutput, error)
PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error)
GetObject(ctx context.Context, bucket, key, versionID, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string) (*s3.GetObjectOutput, error)
PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error)
}
42 changes: 28 additions & 14 deletions s3proxy/internal/router/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,30 +74,36 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) {
bucket := parts[0]

if req.Header.Get("Range") != "" {
r.log.Error("GetObject Range header unsupported")
r.log.Errorf("GetObject Range header unsupported")
http.Error(w, "s3proxy currently does not support Range headers", http.StatusNotImplemented)
}

obj := object{
client: client,
key: key,
bucket: bucket,
query: req.URL.Query(),
log: r.log,
client: client,
key: key,
bucket: bucket,
query: req.URL.Query(),
sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"),
sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"),
sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"),
log: r.log,
}
h = get(obj.get)
case !containsBucket(req.Host) && match(path, "/([^/?]+)/(.+)", &bucket, &key) && req.Method == "GET" && !isGetObjectX(req.URL.Query()):
if req.Header.Get("Range") != "" {
r.log.Error("GetObject Range header unsupported")
r.log.Errorf("GetObject Range header unsupported")
http.Error(w, "s3proxy currently does not support Range headers", http.StatusNotImplemented)
}

obj := object{
client: client,
key: key,
bucket: bucket,
query: req.URL.Query(),
log: r.log,
client: client,
key: key,
bucket: bucket,
query: req.URL.Query(),
sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"),
sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"),
sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"),
log: r.log,
}
h = get(obj.get)

Expand All @@ -122,7 +128,8 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) {
// If the client intentionally sends a mismatching content digest, we would take the client request, rewrap it,
// calculate the correct digest for the new body and NOT get an error.
// Thus we have to check incoming requets for matching content digests.
if clientDigest != "" && clientDigest != serverDigest {
// UNSIGNED-PAYLOAD can be used to disabled payload signing. In that case we don't check the content digest.
if clientDigest != "" && clientDigest != "UNSIGNED-PAYLOAD" && clientDigest != serverDigest {
r.log.Debugf("PutObject", "error", "x-amz-content-sha256 mismatch")
// The S3 API responds with an XML formatted error message.
mismatchErr := NewContentSHA256MismatchError(clientDigest, serverDigest)
Expand Down Expand Up @@ -166,6 +173,9 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) {
objectLockLegalHoldStatus: req.Header.Get("x-amz-object-lock-legal-hold"),
objectLockMode: req.Header.Get("x-amz-object-lock-mode"),
objectLockRetainUntilDate: retentionTime,
sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"),
sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"),
sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"),
log: r.log,
}

Expand All @@ -188,7 +198,8 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) {
// If the client intentionally sends a mismatching content digest, we would take the client request, rewrap it,
// calculate the correct digest for the new body and NOT get an error.
// Thus we have to check incoming requets for matching content digests.
if clientDigest != "" && clientDigest != serverDigest {
// UNSIGNED-PAYLOAD can be used to disabled payload signing. In that case we don't check the content digest.
if clientDigest != "" && clientDigest != "UNSIGNED-PAYLOAD" && clientDigest != serverDigest {
r.log.Debugf("PutObject", "error", "x-amz-content-sha256 mismatch")
// The S3 API responds with an XML formatted error message.
mismatchErr := NewContentSHA256MismatchError(clientDigest, serverDigest)
Expand Down Expand Up @@ -232,6 +243,9 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) {
objectLockLegalHoldStatus: req.Header.Get("x-amz-object-lock-legal-hold"),
objectLockMode: req.Header.Get("x-amz-object-lock-mode"),
objectLockRetainUntilDate: retentionTime,
sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"),
sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"),
sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"),
log: r.log,
}

Expand Down
22 changes: 20 additions & 2 deletions s3proxy/internal/s3/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,21 +48,30 @@ func NewClient(region string) (*Client, error) {

// GetObject returns the object with the given key from the given bucket.
// If a versionID is given, the specific version of the object is returned.
func (c Client) GetObject(ctx context.Context, bucket, key, versionID string) (*s3.GetObjectOutput, error) {
func (c Client) GetObject(ctx context.Context, bucket, key, versionID, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string) (*s3.GetObjectOutput, error) {
getObjectInput := &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
}
if versionID != "" {
getObjectInput.VersionId = &versionID
}
if sseCustomerAlgorithm != "" {
getObjectInput.SSECustomerAlgorithm = &sseCustomerAlgorithm
}
if sseCustomerKey != "" {
getObjectInput.SSECustomerKey = &sseCustomerKey
}
if sseCustomerKeyMD5 != "" {
getObjectInput.SSECustomerKeyMD5 = &sseCustomerKeyMD5
}

return c.s3client.GetObject(ctx, getObjectInput)
}

// PutObject creates a new object in the given bucket with the given key and body.
// Various optional parameters can be set.
func (c Client) PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error) {
func (c Client) PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error) {
// The AWS Go SDK has two versions. V1 does not set the Content-Type header.
// V2 always sets the Content-Type header. We use V2.
// The s3 API sets an object's content-type to binary/octet-stream if
Expand All @@ -87,6 +96,15 @@ func (c Client) PutObject(ctx context.Context, bucket, key, tags, contentType, o
ContentType: &contentType,
ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatus(objectLockLegalHoldStatus),
}
if sseCustomerAlgorithm != "" {
putObjectInput.SSECustomerAlgorithm = &sseCustomerAlgorithm
}
if sseCustomerKey != "" {
putObjectInput.SSECustomerKey = &sseCustomerKey
}
if sseCustomerKeyMD5 != "" {
putObjectInput.SSECustomerKeyMD5 = &sseCustomerKeyMD5
}

// It is not allowed to only set one of these two properties.
if objectLockMode != "" && !objectLockRetainUntilDate.IsZero() {
Expand Down

0 comments on commit bf24452

Please sign in to comment.