From bf24452d6dec2a2b896282219c7b0763c3c78578 Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Fri, 29 Sep 2023 10:03:57 +0200 Subject: [PATCH] s3proxy: handle server side encryption (sse) Also handle unsigned payloads. And correctly return ETags. --- s3proxy/internal/router/object.go | 48 +++++++++++++++++++++++++------ s3proxy/internal/router/router.go | 42 ++++++++++++++++++--------- s3proxy/internal/s3/s3.go | 22 ++++++++++++-- 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/s3proxy/internal/router/object.go b/s3proxy/internal/router/object.go index a9a09d423a..a4ad3a685c 100644 --- a/s3proxy/internal/router/object.go +++ b/s3proxy/internal/router/object.go @@ -42,6 +42,9 @@ type object struct { objectLockLegalHoldStatus string objectLockMode string objectLockRetainUntilDate time.Time + sseCustomerAlgorithm string + sseCustomerKey string + sseCustomerKeyMD5 string log *logger.Logger } @@ -54,7 +57,7 @@ func (o object) get(w http.ResponseWriter, r *http.Request) { versionID = []string{""} } - data, err := o.client.GetObject(r.Context(), o.bucket, o.key, versionID[0]) + output, err := o.client.GetObject(r.Context(), o.bucket, o.key, versionID[0], o.sseCustomerAlgorithm, o.sseCustomerKey, o.sseCustomerKeyMD5) if err != nil { // log with Info as it might be expected behavior (e.g. object not found). o.log.Errorf("GetObject sending request to S3", "error", err) @@ -70,11 +73,38 @@ func (o object) get(w http.ResponseWriter, r *http.Request) { return } - if data.ETag != nil { - w.Header().Set("ETag", strings.Trim(*data.ETag, "\"")) + if output.ETag != nil { + w.Header().Set("ETag", strings.Trim(*output.ETag, "\"")) + } + if output.Expiration != nil { + w.Header().Set("x-amz-expiration", *output.Expiration) + } + if output.ChecksumCRC32 != nil { + w.Header().Set("x-amz-checksum-crc32", *output.ChecksumCRC32) + } + if output.ChecksumCRC32C != nil { + w.Header().Set("x-amz-checksum-crc32c", *output.ChecksumCRC32C) + } + if output.ChecksumSHA1 != nil { + w.Header().Set("x-amz-checksum-sha1", *output.ChecksumSHA1) + } + if output.ChecksumSHA256 != nil { + w.Header().Set("x-amz-checksum-sha256", *output.ChecksumSHA256) + } + if output.SSECustomerAlgorithm != nil { + w.Header().Set("x-amz-server-side-encryption-customer-algorithm", *output.SSECustomerAlgorithm) + } + if output.SSECustomerKeyMD5 != nil { + w.Header().Set("x-amz-server-side-encryption-customer-key-MD5", *output.SSECustomerKeyMD5) + } + if output.SSEKMSKeyId != nil { + w.Header().Set("x-amz-server-side-encryption-aws-kms-key-id", *output.SSEKMSKeyId) + } + if output.ServerSideEncryption != "" { + w.Header().Set("x-amz-server-side-encryption-context", string(output.ServerSideEncryption)) } - body, err := io.ReadAll(data.Body) + body, err := io.ReadAll(output.Body) if err != nil { o.log.Errorf("GetObject reading S3 response", "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) @@ -82,7 +112,7 @@ func (o object) get(w http.ResponseWriter, r *http.Request) { } plaintext := body - decrypt, ok := data.Metadata[encryptionTag] + decrypt, ok := output.Metadata[encryptionTag] if ok && decrypt == "true" { plaintext, err = crypto.Decrypt(body, []byte(testingKey)) @@ -111,7 +141,7 @@ func (o object) put(w http.ResponseWriter, r *http.Request) { // GetObject needs to be able to recognize these objects and skip decryption. o.metadata[encryptionTag] = "true" - output, err := o.client.PutObject(r.Context(), o.bucket, o.key, o.tags, o.contentType, o.objectLockLegalHoldStatus, o.objectLockMode, o.objectLockRetainUntilDate, o.metadata, ciphertext) + output, err := o.client.PutObject(r.Context(), o.bucket, o.key, o.tags, o.contentType, o.objectLockLegalHoldStatus, o.objectLockMode, o.sseCustomerAlgorithm, o.sseCustomerKey, o.sseCustomerKeyMD5, o.objectLockRetainUntilDate, o.metadata, ciphertext) if err != nil { o.log.Errorf("PutObject sending request to S3", "error", err) @@ -132,7 +162,7 @@ func (o object) put(w http.ResponseWriter, r *http.Request) { w.Header().Set("x-amz-version-id", *output.VersionId) } if output.ETag != nil { - w.Header().Set("ETag", *output.ETag) + w.Header().Set("ETag", strings.Trim(*output.ETag, "\"")) } if output.Expiration != nil { w.Header().Set("x-amz-expiration", *output.Expiration) @@ -180,6 +210,6 @@ func parseErrorCode(err error) int { } type s3Client interface { - GetObject(ctx context.Context, bucket, key, versionID string) (*s3.GetObjectOutput, error) - PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error) + GetObject(ctx context.Context, bucket, key, versionID, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string) (*s3.GetObjectOutput, error) + PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error) } diff --git a/s3proxy/internal/router/router.go b/s3proxy/internal/router/router.go index 4c8b8abaa9..b68daed669 100644 --- a/s3proxy/internal/router/router.go +++ b/s3proxy/internal/router/router.go @@ -74,30 +74,36 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) { bucket := parts[0] if req.Header.Get("Range") != "" { - r.log.Error("GetObject Range header unsupported") + r.log.Errorf("GetObject Range header unsupported") http.Error(w, "s3proxy currently does not support Range headers", http.StatusNotImplemented) } obj := object{ - client: client, - key: key, - bucket: bucket, - query: req.URL.Query(), - log: r.log, + client: client, + key: key, + bucket: bucket, + query: req.URL.Query(), + sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"), + sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"), + sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"), + log: r.log, } h = get(obj.get) case !containsBucket(req.Host) && match(path, "/([^/?]+)/(.+)", &bucket, &key) && req.Method == "GET" && !isGetObjectX(req.URL.Query()): if req.Header.Get("Range") != "" { - r.log.Error("GetObject Range header unsupported") + r.log.Errorf("GetObject Range header unsupported") http.Error(w, "s3proxy currently does not support Range headers", http.StatusNotImplemented) } obj := object{ - client: client, - key: key, - bucket: bucket, - query: req.URL.Query(), - log: r.log, + client: client, + key: key, + bucket: bucket, + query: req.URL.Query(), + sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"), + sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"), + sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"), + log: r.log, } h = get(obj.get) @@ -122,7 +128,8 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) { // If the client intentionally sends a mismatching content digest, we would take the client request, rewrap it, // calculate the correct digest for the new body and NOT get an error. // Thus we have to check incoming requets for matching content digests. - if clientDigest != "" && clientDigest != serverDigest { + // UNSIGNED-PAYLOAD can be used to disabled payload signing. In that case we don't check the content digest. + if clientDigest != "" && clientDigest != "UNSIGNED-PAYLOAD" && clientDigest != serverDigest { r.log.Debugf("PutObject", "error", "x-amz-content-sha256 mismatch") // The S3 API responds with an XML formatted error message. mismatchErr := NewContentSHA256MismatchError(clientDigest, serverDigest) @@ -166,6 +173,9 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) { objectLockLegalHoldStatus: req.Header.Get("x-amz-object-lock-legal-hold"), objectLockMode: req.Header.Get("x-amz-object-lock-mode"), objectLockRetainUntilDate: retentionTime, + sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"), + sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"), + sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"), log: r.log, } @@ -188,7 +198,8 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) { // If the client intentionally sends a mismatching content digest, we would take the client request, rewrap it, // calculate the correct digest for the new body and NOT get an error. // Thus we have to check incoming requets for matching content digests. - if clientDigest != "" && clientDigest != serverDigest { + // UNSIGNED-PAYLOAD can be used to disabled payload signing. In that case we don't check the content digest. + if clientDigest != "" && clientDigest != "UNSIGNED-PAYLOAD" && clientDigest != serverDigest { r.log.Debugf("PutObject", "error", "x-amz-content-sha256 mismatch") // The S3 API responds with an XML formatted error message. mismatchErr := NewContentSHA256MismatchError(clientDigest, serverDigest) @@ -232,6 +243,9 @@ func (r Router) Serve(w http.ResponseWriter, req *http.Request) { objectLockLegalHoldStatus: req.Header.Get("x-amz-object-lock-legal-hold"), objectLockMode: req.Header.Get("x-amz-object-lock-mode"), objectLockRetainUntilDate: retentionTime, + sseCustomerAlgorithm: req.Header.Get("x-amz-server-side-encryption-customer-algorithm"), + sseCustomerKey: req.Header.Get("x-amz-server-side-encryption-customer-key"), + sseCustomerKeyMD5: req.Header.Get("x-amz-server-side-encryption-customer-key-MD5"), log: r.log, } diff --git a/s3proxy/internal/s3/s3.go b/s3proxy/internal/s3/s3.go index 29b8ee7478..462530be73 100644 --- a/s3proxy/internal/s3/s3.go +++ b/s3proxy/internal/s3/s3.go @@ -48,7 +48,7 @@ func NewClient(region string) (*Client, error) { // GetObject returns the object with the given key from the given bucket. // If a versionID is given, the specific version of the object is returned. -func (c Client) GetObject(ctx context.Context, bucket, key, versionID string) (*s3.GetObjectOutput, error) { +func (c Client) GetObject(ctx context.Context, bucket, key, versionID, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string) (*s3.GetObjectOutput, error) { getObjectInput := &s3.GetObjectInput{ Bucket: &bucket, Key: &key, @@ -56,13 +56,22 @@ func (c Client) GetObject(ctx context.Context, bucket, key, versionID string) (* if versionID != "" { getObjectInput.VersionId = &versionID } + if sseCustomerAlgorithm != "" { + getObjectInput.SSECustomerAlgorithm = &sseCustomerAlgorithm + } + if sseCustomerKey != "" { + getObjectInput.SSECustomerKey = &sseCustomerKey + } + if sseCustomerKeyMD5 != "" { + getObjectInput.SSECustomerKeyMD5 = &sseCustomerKeyMD5 + } return c.s3client.GetObject(ctx, getObjectInput) } // PutObject creates a new object in the given bucket with the given key and body. // Various optional parameters can be set. -func (c Client) PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error) { +func (c Client) PutObject(ctx context.Context, bucket, key, tags, contentType, objectLockLegalHoldStatus, objectLockMode, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMD5 string, objectLockRetainUntilDate time.Time, metadata map[string]string, body []byte) (*s3.PutObjectOutput, error) { // The AWS Go SDK has two versions. V1 does not set the Content-Type header. // V2 always sets the Content-Type header. We use V2. // The s3 API sets an object's content-type to binary/octet-stream if @@ -87,6 +96,15 @@ func (c Client) PutObject(ctx context.Context, bucket, key, tags, contentType, o ContentType: &contentType, ObjectLockLegalHoldStatus: types.ObjectLockLegalHoldStatus(objectLockLegalHoldStatus), } + if sseCustomerAlgorithm != "" { + putObjectInput.SSECustomerAlgorithm = &sseCustomerAlgorithm + } + if sseCustomerKey != "" { + putObjectInput.SSECustomerKey = &sseCustomerKey + } + if sseCustomerKeyMD5 != "" { + putObjectInput.SSECustomerKeyMD5 = &sseCustomerKeyMD5 + } // It is not allowed to only set one of these two properties. if objectLockMode != "" && !objectLockRetainUntilDate.IsZero() {