Merge pull request #645 from rvrangel/s3-min-part-size

add s3 minimum part size support
planetscale · Dec 26, 2024 · 4c7ea9c · 4c7ea9c
2 parents b314fac + c999a42
commit 4c7ea9c
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 0 deletions.
diff --git a/deploy/crds/planetscale.com_vitessbackupstorages.yaml b/deploy/crds/planetscale.com_vitessbackupstorages.yaml
@@ -129,6 +129,9 @@ spec:
                         maxLength: 256
                         pattern: ^[^\r\n]*$
                         type: string
+                      minPartSize:
+                        format: int64
+                        type: integer
                       region:
                         minLength: 1
                         type: string

diff --git a/deploy/crds/planetscale.com_vitessclusters.yaml b/deploy/crds/planetscale.com_vitessclusters.yaml
@@ -138,6 +138,9 @@ spec:
                               maxLength: 256
                               pattern: ^[^\r\n]*$
                               type: string
+                            minPartSize:
+                              format: int64
+                              type: integer
                             region:
                               minLength: 1
                               type: string

diff --git a/deploy/crds/planetscale.com_vitesskeyspaces.yaml b/deploy/crds/planetscale.com_vitesskeyspaces.yaml
@@ -136,6 +136,9 @@ spec:
                           maxLength: 256
                           pattern: ^[^\r\n]*$
                           type: string
+                        minPartSize:
+                          format: int64
+                          type: integer
                         region:
                           minLength: 1
                           type: string

diff --git a/deploy/crds/planetscale.com_vitessshards.yaml b/deploy/crds/planetscale.com_vitessshards.yaml
@@ -136,6 +136,9 @@ spec:
                           maxLength: 256
                           pattern: ^[^\r\n]*$
                           type: string
+                        minPartSize:
+                          format: int64
+                          type: integer
                         region:
                           minLength: 1
                           type: string

diff --git a/docs/api.md b/docs/api.md
@@ -1926,6 +1926,20 @@ If set, this must point to a file in the format expected for the
 Default: Use the default credentials of the Node.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>minPartSize</code><br>
+<em>
+int64
+</em>
+</td>
+<td>
+<p>MinPartSize is optional and is only needed if we want to set a minimum part size
+to be used by the S3 uploader, mainly used to avoid too many small requests to S3
+when there are too many small shards.
+Default: 5242880 (5MiB)</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="planetscale.com/v2.SecretSource">SecretSource

diff --git a/docs/api/index.html b/docs/api/index.html
@@ -1928,6 +1928,20 @@ <h3 id="planetscale.com/v2.S3BackupLocation">S3BackupLocation
 Default: Use the default credentials of the Node.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>minPartSize</code><br>
+<em>
+int64
+</em>
+</td>
+<td>
+<p>MinPartSize is optional and is only needed if we want to set a minimum part size
+to be used by the S3 uploader, mainly used to avoid too many small requests to S3
+when there are too many small shards.
+Default: 5242880 (5MiB)</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="planetscale.com/v2.SecretSource">SecretSource

diff --git a/pkg/apis/planetscale/v2/vitessbackupstorage_types.go b/pkg/apis/planetscale/v2/vitessbackupstorage_types.go
@@ -143,6 +143,11 @@ type S3BackupLocation struct {
 	// `~/.aws/credentials` file.
 	// Default: Use the default credentials of the Node.
 	AuthSecret *SecretSource `json:"authSecret,omitempty"`
+	// MinPartSize is optional and is only needed if we want to set a minimum part size
+	// to be used by the S3 uploader, mainly used to avoid too many small requests to S3
+	// when there are too many small shards.
+	// Default: 5242880 (5MiB)
+	MinPartSize int64 `json:"minPartSize,omitempty"`
 }
 
 // AzblobBackupLocation specifies a backup location in Azure Blob Storage.

diff --git a/pkg/operator/vitessbackup/storage_s3.go b/pkg/operator/vitessbackup/storage_s3.go
@@ -31,6 +31,7 @@ func s3BackupFlags(s3 *planetscalev2.S3BackupLocation, clusterName string) vites
 		"s3_backup_storage_bucket":      s3.Bucket,
 		"s3_backup_storage_root":        rootKeyPrefix(s3.KeyPrefix, clusterName),
 		"s3_backup_force_path_style":    s3.ForcePathStyle,
+		"s3_backup_aws_min_partsize":    s3.MinPartSize,
 	}
 	if len(s3.Endpoint) > 0 {
 		flags["s3_backup_aws_endpoint"] = s3.Endpoint

diff --git a/test/endtoend/operator/operator-latest.yaml b/test/endtoend/operator/operator-latest.yaml
@@ -679,6 +679,9 @@ spec:
                           maxLength: 256
                           pattern: ^[^\r\n]*$
                           type: string
+                        minPartSize:
+                          format: int64
+                          type: integer
                         region:
                           minLength: 1
                           type: string
@@ -1995,6 +1998,9 @@ spec:
                                 maxLength: 256
                                 pattern: ^[^\r\n]*$
                                 type: string
+                              minPartSize:
+                                format: int64
+                                type: integer
                               region:
                                 minLength: 1
                                 type: string
@@ -5249,6 +5255,9 @@ spec:
                             maxLength: 256
                             pattern: ^[^\r\n]*$
                             type: string
+                          minPartSize:
+                            format: int64
+                            type: integer
                           region:
                             minLength: 1
                             type: string
@@ -6696,6 +6705,9 @@ spec:
                             maxLength: 256
                             pattern: ^[^\r\n]*$
                             type: string
+                          minPartSize:
+                            format: int64
+                            type: integer
                           region:
                             minLength: 1
                             type: string