fix 4MB index-cache.yaml limit with Azure Blob Storage (#667)

* fix 4MB index-cache.yaml limit with Azure Blob Storage * Update microsoft.go Co-authored-by: Casey Buto <[email protected]> * Update microsoft.go change var name to content --------- Co-authored-by: jumao.yuan <[email protected]> Co-authored-by: Casey Buto <[email protected]>
chartmuseum · Jun 26, 2023 · 3bb41a8 · 3bb41a8
1 parent 0e1e430
commit 3bb41a8
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/microsoft.go b/microsoft.go
@@ -27,6 +27,10 @@ import (
 	microsoft_storage "github.com/Azure/azure-sdk-for-go/storage"
 )
 
+const (
+	maxChunkSize = 4 * 1024 * 1024
+)
+
 // MicrosoftBlobBackend is a storage backend for Microsoft Azure Blob Storage
 type MicrosoftBlobBackend struct {
 	Prefix    string
@@ -155,12 +159,25 @@ func (b MicrosoftBlobBackend) PutObject(path string, content []byte) error {
 
 	err := blobReference.PutAppendBlob(nil)
 	if err == nil {
-		err = blobReference.AppendBlock(content, nil)
+		err = writeToBlob(content, blobReference)
 	}
 
 	return err
 }
 
+func writeToBlob(content []byte, blobRef *microsoft_storage.Blob) error {
+	for offset := 0; offset < len(content); offset += maxChunkSize {
+		chunkSize := maxChunkSize
+		if offset+chunkSize > len(content) {
+			chunkSize = len(content) - offset
+		}
+		if err := blobRef.AppendBlock(content[offset:offset+chunkSize], nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // DeleteObject removes an object from Microsoft Azure Blob Storage container, at path
 func (b MicrosoftBlobBackend) DeleteObject(path string) error {
 	if b.Container == nil {

diff --git a/microsoft_test.go b/microsoft_test.go
@@ -89,6 +89,17 @@ func (suite *MicrosoftTestSuite) TestPutObject() {
 	suite.NotNil(err, "cannot put objects with bad bucket")
 }
 
+func (suite *MicrosoftTestSuite) TestPutObjectGreaterThan4MB() {
+	data := make([]byte, 2*maxChunkSize)
+	for i := 0; i < 2*maxChunkSize; i++ {
+		data[i] = byte(i)
+	}
+	err := suite.NoPrefixAzureBlobBackend.PutObject("this-file-created-for-test.txt", data)
+	suite.Nil(err, "can put objects with good bucket, no prefix")
+	// clean up
+	suite.NoPrefixAzureBlobBackend.DeleteObject("this-file-created-for-test.txt")
+}
+
 func TestAzureStorageTestSuite(t *testing.T) {
 	if os.Getenv("TEST_CLOUD_STORAGE") == "1" &&
 		os.Getenv("TEST_STORAGE_AZURE_CONTAINER") != "" {