fix ci lint and improve client serialization

hasura · Jan 6, 2025 · ca97903 · ca97903
1 parent 964034a
commit ca97903
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 16 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -38,7 +38,6 @@ jobs:
       - name: Format
         run: |
           diff -u <(echo -n) <(gofmt -d -s .)
-          cd ndc-http-schema && diff -u <(echo -n) <(gofmt -d -s .)
       - name: golangci-lint ${{ matrix.modules }}
         uses: golangci/golangci-lint-action@v6
         with:

diff --git a/connector/storage/common/storage.go b/connector/storage/common/storage.go
@@ -153,14 +153,14 @@ type StorageObject struct {
 	// An ETag is optionally set to md5sum of an object.  In case of multipart objects,
 	// ETag is of the form MD5SUM-N where MD5SUM is md5sum of all individual md5sums of
 	// each parts concatenated into one string.
-	ETag string `json:"etag"`
+	ETag *string `json:"etag"`
 
-	Bucket       string    `json:"bucket"`       // Name of the bucket
-	Name         string    `json:"name"`         // Name of the object
-	LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
-	Size         int64     `json:"size"`         // Size in bytes of the object.
-	ContentType  string    `json:"contentType"`  // A standard MIME type describing the format of the object data.
-	Expires      time.Time `json:"expires"`      // The date and time at which the object is no longer able to be cached.
+	Bucket       string     `json:"bucket"`       // Name of the bucket
+	Name         string     `json:"name"`         // Name of the object
+	LastModified time.Time  `json:"lastModified"` // Date and time the object was last modified.
+	Size         int64      `json:"size"`         // Size in bytes of the object.
+	ContentType  *string    `json:"contentType"`  // A standard MIME type describing the format of the object data.
+	Expires      *time.Time `json:"expires"`      // The date and time at which the object is no longer able to be cached.
 
 	// Collection of additional metadata on the object.
 	// eg: x-amz-meta-*, content-encoding etc.

diff --git a/connector/storage/common/telemetry.go b/connector/storage/common/telemetry.go
@@ -40,8 +40,8 @@ func SetObjectInfoSpanAttributes(span trace.Span, object *StorageObject) {
 	span.SetAttributes(attribute.Int64("storage.object.size", object.Size))
 	SetObjectChecksumSpanAttributes(span, &object.StorageObjectChecksum)
 
-	if object.ETag != "" {
-		span.SetAttributes(attribute.String("storage.object.etag", object.ETag))
+	if object.ETag != nil {
+		span.SetAttributes(attribute.String("storage.object.etag", *object.ETag))
 	}
 
 	if object.StorageClass != nil {

diff --git a/connector/storage/minio/object.go b/connector/storage/minio/object.go
@@ -22,7 +22,7 @@ func (mc *Client) ListObjects(ctx context.Context, opts *common.ListStorageObjec
 	ctx, span := mc.startOtelSpan(ctx, "ListObjects", opts.Bucket)
 	defer span.End()
 
-	objChan := mc.client.ListObjects(ctx, opts.Bucket, serializeListObjectsOptions(span, opts))
+	objChan := mc.client.ListObjects(ctx, opts.Bucket, mc.validateListObjectsOptions(span, opts))
 	objects := make([]common.StorageObject, 0)
 
 	for obj := range objChan {
@@ -338,7 +338,7 @@ func (mc *Client) RemoveObjects(ctx context.Context, opts *common.RemoveStorageO
 	ctx, span := mc.startOtelSpan(ctx, "RemoveObjects", opts.Bucket)
 	defer span.End()
 
-	listOptions := serializeListObjectsOptions(span, &opts.ListStorageObjectsOptions)
+	listOptions := mc.validateListObjectsOptions(span, &opts.ListStorageObjectsOptions)
 	span.SetAttributes(attribute.Bool("storage.options.governance_bypass", opts.GovernanceBypass))
 
 	objectChan := mc.client.ListObjects(ctx, opts.Bucket, listOptions)

diff --git a/connector/storage/minio/utils.go b/connector/storage/minio/utils.go
@@ -64,12 +64,9 @@ func serializeObjectInfo(obj minio.ObjectInfo) common.StorageObject {
 	}
 
 	object := common.StorageObject{
-		ETag:                  obj.ETag,
 		Name:                  obj.Key,
 		LastModified:          obj.LastModified,
 		Size:                  obj.Size,
-		ContentType:           obj.ContentType,
-		Expires:               obj.Expires,
 		Metadata:              obj.Metadata,
 		UserMetadata:          obj.UserMetadata,
 		UserTags:              obj.UserTags,
@@ -81,6 +78,18 @@ func serializeObjectInfo(obj minio.ObjectInfo) common.StorageObject {
 		StorageObjectChecksum: checksum,
 	}
 
+	if !isStringNull(obj.ETag) {
+		object.ETag = &obj.ETag
+	}
+
+	if !isStringNull(obj.ContentType) {
+		object.ContentType = &obj.ContentType
+	}
+
+	if !obj.Expires.IsZero() {
+		object.Expires = &obj.Expires
+	}
+
 	if !isStringNull(obj.Owner.DisplayName) || !isStringNull(obj.Owner.ID) {
 		object.Owner = &common.StorageOwner{}
 		if !isStringNull(obj.Owner.DisplayName) {
@@ -126,7 +135,12 @@ func serializeObjectInfo(obj minio.ObjectInfo) common.StorageObject {
 	return object
 }
 
-func serializeListObjectsOptions(span trace.Span, opts *common.ListStorageObjectsOptions) minio.ListObjectsOptions {
+func (mc *Client) validateListObjectsOptions(span trace.Span, opts *common.ListStorageObjectsOptions) minio.ListObjectsOptions {
+	if mc.providerType == common.GoogleStorage && opts.WithVersions {
+		// Force versioning off. GCS doesn't support AWS S3 compatible versioning API.
+		opts.WithVersions = false
+	}
+
 	span.SetAttributes(
 		attribute.Bool("storage.options.recursive", opts.Recursive),
 		attribute.Bool("storage.options.with_versions", opts.WithVersions),