increase the partsize for s3 upload

PeerDB-io · May 16, 2024 · ff8d25b · ff8d25b
1 parent bd8308d
commit ff8d25b
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/flow/connectors/utils/avro/avro_writer.go b/flow/connectors/utils/avro/avro_writer.go
@@ -197,7 +197,11 @@ func (p *peerDBOCFWriter) WriteRecordsToS3(
 		numRows, writeOcfError = p.WriteOCF(ctx, w)
 	}()
 
-	_, err = manager.NewUploader(s3svc).Upload(ctx, &s3.PutObjectInput{
+	uploader := manager.NewUploader(s3svc, func(u *manager.Uploader) {
+		u.PartSize = 4 * 1024 * 1024 * 1024
+	})
+
+	_, err = uploader.Upload(ctx, &s3.PutObjectInput{
 		Bucket: aws.String(bucketName),
 		Key:    aws.String(key),
 		Body:   r,