From 3bb41a85842c0ac223a258e424de0d6f0daf7212 Mon Sep 17 00:00:00 2001
From: yuanjumao <492009455@qq.com>
Date: Tue, 27 Jun 2023 01:55:11 +0800
Subject: [PATCH] fix 4MB index-cache.yaml limit with Azure Blob Storage (#667)

* fix 4MB index-cache.yaml limit with Azure Blob Storage

* Update microsoft.go

Co-authored-by: Casey Buto <cbuto22@gmail.com>

* Update microsoft.go

change var name to content

---------

Co-authored-by: jumao.yuan <jumao.yuan@envision-digital.com>
Co-authored-by: Casey Buto <cbuto22@gmail.com>
---
 microsoft.go      | 19 ++++++++++++++++++-
 microsoft_test.go | 11 +++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/microsoft.go b/microsoft.go
index 92816e4f..bbe52bb1 100644
--- a/microsoft.go
+++ b/microsoft.go
@@ -27,6 +27,10 @@ import (
 	microsoft_storage "github.com/Azure/azure-sdk-for-go/storage"
 )
 
+const (
+	maxChunkSize = 4 * 1024 * 1024
+)
+
 // MicrosoftBlobBackend is a storage backend for Microsoft Azure Blob Storage
 type MicrosoftBlobBackend struct {
 	Prefix    string
@@ -155,12 +159,25 @@ func (b MicrosoftBlobBackend) PutObject(path string, content []byte) error {
 
 	err := blobReference.PutAppendBlob(nil)
 	if err == nil {
-		err = blobReference.AppendBlock(content, nil)
+		err = writeToBlob(content, blobReference)
 	}
 
 	return err
 }
 
+func writeToBlob(content []byte, blobRef *microsoft_storage.Blob) error {
+	for offset := 0; offset < len(content); offset += maxChunkSize {
+		chunkSize := maxChunkSize
+		if offset+chunkSize > len(content) {
+			chunkSize = len(content) - offset
+		}
+		if err := blobRef.AppendBlock(content[offset:offset+chunkSize], nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // DeleteObject removes an object from Microsoft Azure Blob Storage container, at path
 func (b MicrosoftBlobBackend) DeleteObject(path string) error {
 	if b.Container == nil {
diff --git a/microsoft_test.go b/microsoft_test.go
index 0f2e6fcb..0af77e46 100644
--- a/microsoft_test.go
+++ b/microsoft_test.go
@@ -89,6 +89,17 @@ func (suite *MicrosoftTestSuite) TestPutObject() {
 	suite.NotNil(err, "cannot put objects with bad bucket")
 }
 
+func (suite *MicrosoftTestSuite) TestPutObjectGreaterThan4MB() {
+	data := make([]byte, 2*maxChunkSize)
+	for i := 0; i < 2*maxChunkSize; i++ {
+		data[i] = byte(i)
+	}
+	err := suite.NoPrefixAzureBlobBackend.PutObject("this-file-created-for-test.txt", data)
+	suite.Nil(err, "can put objects with good bucket, no prefix")
+	// clean up
+	suite.NoPrefixAzureBlobBackend.DeleteObject("this-file-created-for-test.txt")
+}
+
 func TestAzureStorageTestSuite(t *testing.T) {
 	if os.Getenv("TEST_CLOUD_STORAGE") == "1" &&
 		os.Getenv("TEST_STORAGE_AZURE_CONTAINER") != "" {