diff --git a/connector/connector.go b/connector/connector.go index 3c77d15..cd8aaf2 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -11,7 +11,6 @@ import ( "github.com/hasura/ndc-sdk-go/connector" "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-sdk-go/utils" - "github.com/hasura/ndc-storage/configuration/version" "github.com/hasura/ndc-storage/connector/functions" "github.com/hasura/ndc-storage/connector/storage" "github.com/hasura/ndc-storage/connector/types" @@ -73,7 +72,7 @@ func (c *Connector) ParseConfiguration(ctx context.Context, configurationDir str func (c *Connector) TryInitState(ctx context.Context, configuration *types.Configuration, metrics *connector.TelemetryState) (*types.State, error) { logger := connector.GetLogger(ctx) - manager, err := storage.NewManager(ctx, configuration.Clients, logger, version.BuildVersion) + manager, err := storage.NewManager(ctx, configuration.Clients, logger) if err != nil { return nil, err } diff --git a/connector/connector_test.go b/connector/connector_test.go index c08e3f5..b976fb5 100644 --- a/connector/connector_test.go +++ b/connector/connector_test.go @@ -9,6 +9,17 @@ import ( ) func TestConnector(t *testing.T) { + setConnectorTestEnv(t) + + for _, dir := range []string{"01-setup", "02-get", "03-cleanup"} { + ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{ + Configuration: "../tests/configuration", + TestDataDir: filepath.Join("testdata", dir), + }) + } +} + +func setConnectorTestEnv(t *testing.T) { azureBlobEndpoint := "http://local.hasura.dev:10000" azureAccountName := "local" azureAccountKey := "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" @@ -30,11 +41,4 @@ func TestConnector(t *testing.T) { t.Setenv("GOOGLE_PROJECT_ID", "test-local-project") t.Setenv("GOOGLE_STORAGE_ENDPOINT", "http://localhost:10010/storage/v1/") t.Setenv("GOOGLE_STORAGE_CREDENTIALS_FILE", "../tests/certs/service_account.json") - - for _, dir := range []string{"01-setup", "02-get", "03-cleanup"} { - ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{ - Configuration: "../tests/configuration", - TestDataDir: filepath.Join("testdata", dir), - }) - } } diff --git a/connector/functions/bucket.go b/connector/functions/bucket.go index b02c2fa..8086825 100644 --- a/connector/functions/bucket.go +++ b/connector/functions/bucket.go @@ -21,7 +21,7 @@ func ProcedureCreateStorageBucket(ctx context.Context, state *types.State, args // FunctionStorageBuckets list all buckets. func FunctionStorageBuckets(ctx context.Context, state *types.State, args *common.ListStorageBucketArguments) (common.StorageBucketListResults, error) { - if args.MaxResults <= 0 { + if args.MaxResults != nil && *args.MaxResults <= 0 { return common.StorageBucketListResults{}, schema.UnprocessableContentError("maxResults must be larger than 0", nil) } @@ -54,7 +54,7 @@ func FunctionStorageBuckets(ctx context.Context, state *types.State, args *commo Versioning: request.Include.Versions, Lifecycle: request.Include.Lifecycle, Encryption: request.Include.Encryption, - ObjectLock: request.Include.ObjectLock, + ObjectLock: request.IncludeObjectLock, }, NumThreads: state.Concurrency.Query, }, predicate) @@ -79,7 +79,7 @@ func FunctionStorageBucket(ctx context.Context, state *types.State, args *common Versioning: request.Include.Versions, Lifecycle: request.Include.Lifecycle, Encryption: request.Include.Encryption, - ObjectLock: request.Include.ObjectLock, + ObjectLock: request.IncludeObjectLock, }, NumThreads: state.Concurrency.Query, }) diff --git a/connector/functions/internal/predicate.go b/connector/functions/internal/predicate.go index 013518b..e8836e5 100644 --- a/connector/functions/internal/predicate.go +++ b/connector/functions/internal/predicate.go @@ -12,9 +12,10 @@ import ( // PredicateEvaluator the structured predicate result which is evaluated from the raw expression. type PredicateEvaluator struct { - ClientID *common.StorageClientID - IsValid bool - Include common.StorageObjectIncludeOptions + ClientID *common.StorageClientID + IsValid bool + Include common.StorageObjectIncludeOptions + IncludeObjectLock bool variables map[string]any BucketPredicate StringFilterPredicate @@ -137,7 +138,7 @@ func (pe *PredicateEvaluator) EvalSelection(selection schema.NestedField) error } if _, ok := expr.Fields["objectLock"]; ok { - pe.Include.ObjectLock = true + pe.IncludeObjectLock = true } } diff --git a/connector/query_test.go b/connector/query_test.go new file mode 100644 index 0000000..36162e2 --- /dev/null +++ b/connector/query_test.go @@ -0,0 +1,131 @@ +package connector + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "testing" + + "github.com/hasura/ndc-sdk-go/ndctest" + "github.com/hasura/ndc-sdk-go/schema" + "gotest.tools/v3/assert" +) + +func TestConnectorQueries(t *testing.T) { + connectorHost := "http://localhost:8080" + clientIDs := []string{"minio", "azblob", "gcs"} + + for _, cid := range clientIDs { + t.Run("create_bucket_"+cid, func(t *testing.T) { + procedureRequest := schema.MutationRequest{ + CollectionRelationships: schema.MutationRequestCollectionRelationships{}, + Operations: []schema.MutationOperation{}, + } + + for i := range 10 { + procedureRequest.Operations = append(procedureRequest.Operations, schema.MutationOperation{ + Type: schema.MutationOperationProcedure, + Name: "createStorageBucket", + Arguments: []byte(fmt.Sprintf(`{ + "clientId": "%s", + "name": "dummy-bucket-%d" + }`, cid, i)), + }) + } + + rawBody, err := json.Marshal(procedureRequest) + assert.NilError(t, err) + + resp, err := http.DefaultClient.Post(connectorHost+"/mutation", "application/json", bytes.NewReader(rawBody)) + assert.NilError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + } + + objectFixtures := map[string]string{ + "movies/1900s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1900s.json", + "movies/1910s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1910s.json", + "movies/1920s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1920s.json", + "movies/1930s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1930s.json", + "movies/1940s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1940s.json", + "movies/1950s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1950s.json", + "movies/1960s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1960s.json", + "movies/1970s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1970s.json", + "movies/1980s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1980s.json", + "movies/1990s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1990s.json", + "movies/2000s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-2000s.json", + } + + for key, value := range objectFixtures { + resp, err := http.DefaultClient.Get(value) + assert.NilError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + rawBody, err := io.ReadAll(resp.Body) + assert.NilError(t, err) + resp.Body.Close() + + for _, cid := range clientIDs { + t.Run(fmt.Sprintf("upload_object_%s/%s", cid, key), func(t *testing.T) { + arguments := map[string]any{ + "clientId": cid, + "bucket": "dummy-bucket-0", + "data": string(rawBody), + "object": key, + "options": map[string]any{ + "cacheControl": "max-age=100", + "contentDisposition": "attachment", + "contentLanguage": "en-US", + "contentType": "application/json", + "expires": "2099-01-01T00:00:00Z", + "sendContentMd5": true, + "metadata": map[string]any{ + "Foo": "Baz", + }, + "tags": map[string]any{ + "category": "movie", + }, + }, + } + + rawArguments, err := json.Marshal(arguments) + assert.NilError(t, err) + + procedureRequest := schema.MutationRequest{ + CollectionRelationships: schema.MutationRequestCollectionRelationships{}, + Operations: []schema.MutationOperation{ + { + Type: schema.MutationOperationProcedure, + Name: "uploadStorageObjectText", + Arguments: rawArguments, + Fields: schema.NewNestedObject(map[string]schema.FieldEncoder{ + "name": schema.NewColumnField("name", nil), + "size": schema.NewColumnField("size", nil), + }).Encode(), + }, + }, + } + + uploadBytes, err := json.Marshal(procedureRequest) + assert.NilError(t, err) + + resp, err = http.DefaultClient.Post(connectorHost+"/mutation", "application/json", bytes.NewReader(uploadBytes)) + assert.NilError(t, err) + resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + }) + } + } + + setConnectorTestEnv(t) + + for _, dir := range []string{"bucket", "object"} { + ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{ + Configuration: "../tests/configuration", + TestDataDir: filepath.Join("testdata", dir), + }) + } +} diff --git a/connector/schema.generated.go b/connector/schema.generated.go index 87b39fd..362701e 100644 --- a/connector/schema.generated.go +++ b/connector/schema.generated.go @@ -513,6 +513,9 @@ func GetConnectorSchema() *schema.SchemaResponse { "autoclass": schema.ObjectField{ Type: schema.NewNullableType(schema.NewNamedType("BucketAutoclass")).Encode(), }, + "clientId": schema.ObjectField{ + Type: schema.NewNamedType("String").Encode(), + }, "cors": schema.ObjectField{ Type: schema.NewNullableType(schema.NewArrayType(schema.NewNamedType("BucketCors"))).Encode(), }, diff --git a/connector/storage/azblob/bucket.go b/connector/storage/azblob/bucket.go index 5ff926b..f8fdc7b 100644 --- a/connector/storage/azblob/bucket.go +++ b/connector/storage/azblob/bucket.go @@ -56,9 +56,12 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc opts.Prefix = &options.Prefix } - maxResults := int32(options.MaxResults) - if options.MaxResults > 0 && predicate == nil { + var maxResults int32 + if options.MaxResults != nil && *options.MaxResults > 0 && predicate == nil { + maxResults = int32(*options.MaxResults) opts.MaxResults = &maxResults + + span.SetAttributes(attribute.Int("storage.options.max_results", int(maxResults))) } if options.StartAfter != "" { @@ -71,6 +74,7 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc var results []common.StorageBucket pageInfo := common.StoragePaginationInfo{} +L: for pager.More() { resp, err := pager.NextPage(ctx) if err != nil { @@ -80,7 +84,7 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc return nil, serializeErrorResponse(err) } - for _, container := range resp.ContainerItems { + for i, container := range resp.ContainerItems { if container.Name == nil || (predicate != nil && !predicate(*container.Name)) { continue } @@ -136,16 +140,12 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc count++ if maxResults > 0 && count >= maxResults { - if pager.More() { + if i < len(resp.ContainerItems)-1 || pager.More() { pageInfo.HasNextPage = true - pageInfo.Cursor = resp.NextMarker - } - - if resp.Marker != nil && *resp.Marker != "" { - pageInfo.Cursor = resp.Marker + pageInfo.Cursor = &result.Name } - break + break L } } } diff --git a/connector/storage/azblob/config.go b/connector/storage/azblob/config.go index e90fbc2..4c95db8 100644 --- a/connector/storage/azblob/config.go +++ b/connector/storage/azblob/config.go @@ -13,11 +13,9 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel" "github.com/hasura/ndc-sdk-go/utils" "github.com/hasura/ndc-storage/connector/storage/common" "github.com/invopop/jsonschema" - "go.opentelemetry.io/otel" ) var ( @@ -66,7 +64,6 @@ func (cc ClientConfig) toAzureBlobClient(logger *slog.Logger) (*azblob.Client, e IncludeBody: isDebug, }, InsecureAllowCredentialWithHTTP: !useSSL, - TracingProvider: azotel.NewTracingProvider(otel.GetTracerProvider(), nil), Transport: &http.Client{ Transport: transport, }, diff --git a/connector/storage/azblob/object.go b/connector/storage/azblob/object.go index 9fc05aa..4e2d12a 100644 --- a/connector/storage/azblob/object.go +++ b/connector/storage/azblob/object.go @@ -25,23 +25,19 @@ import ( // ListObjects list objects in a bucket. func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { + if opts.Recursive { + return c.listFlatObjects(ctx, bucketName, opts, predicate) + } + + return c.listHierarchyObjects(ctx, bucketName, opts, predicate) +} + +func (c *Client) listFlatObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { ctx, span := c.startOtelSpan(ctx, "ListObjects", bucketName) defer span.End() options := &container.ListBlobsFlatOptions{ - Include: container.ListBlobsInclude{ - Versions: opts.Include.Versions, - Metadata: opts.Include.Metadata, - Tags: opts.Include.Tags, - Copy: opts.Include.Copy, - Snapshots: opts.Include.Snapshots, - LegalHold: opts.Include.LegalHold, - ImmutabilityPolicy: opts.Include.Retention, - Permissions: opts.Include.Permissions, - Deleted: false, - DeletedWithVersions: false, - UncommittedBlobs: false, - }, + Include: makeListBlobsInclude(opts.Include), } if opts.Prefix != "" { @@ -62,6 +58,7 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo pager := c.client.NewListBlobsFlatPager(bucketName, options) pageInfo := common.StoragePaginationInfo{} +L: for pager.More() { resp, err := pager.NextPage(ctx) if err != nil { @@ -71,22 +68,110 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo return nil, serializeErrorResponse(err) } - for _, item := range resp.Segment.BlobItems { + for i, item := range resp.Segment.BlobItems { if item.Name == nil || (predicate != nil && !predicate(*item.Name)) { continue } - objects = append(objects, serializeObjectInfo(item)) + object := serializeObjectInfo(item) + objects = append(objects, object) count++ + + if maxResults > 0 && count >= maxResults { + if i < len(resp.Segment.BlobItems)-1 || pager.More() { + pageInfo.HasNextPage = true + pageInfo.Cursor = &object.Name + } + + break L + } } + } - if maxResults > 0 && count >= maxResults { - if pager.More() { - pageInfo.HasNextPage = true - pageInfo.Cursor = resp.NextMarker + span.SetAttributes(attribute.Int("storage.object_count", int(count))) + + results := &common.StorageObjectListResults{ + Objects: objects, + PageInfo: pageInfo, + } + + return results, nil +} + +func (c *Client) listHierarchyObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { //nolint:gocognit,cyclop + ctx, span := c.startOtelSpan(ctx, "ListObjects", bucketName) + defer span.End() + + options := &container.ListBlobsHierarchyOptions{ + Include: makeListBlobsInclude(opts.Include), + } + + if opts.Prefix != "" { + options.Prefix = &opts.Prefix + } + + maxResults := int32(opts.MaxResults) + if opts.MaxResults > 0 && predicate == nil { + options.MaxResults = &maxResults + } + + if opts.StartAfter != "" { + options.Marker = &opts.StartAfter + } + + var count int32 + objects := make([]common.StorageObject, 0) + pager := c.client.ServiceClient().NewContainerClient(bucketName).NewListBlobsHierarchyPager("/", options) + pageInfo := common.StoragePaginationInfo{} + +L: + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + + return nil, serializeErrorResponse(err) + } + + for i, item := range resp.Segment.BlobPrefixes { + if item.Name == nil || (predicate != nil && !predicate(*item.Name)) { + continue } - break + object := common.StorageObject{ + Name: *item.Name, + } + objects = append(objects, object) + count++ + + if maxResults > 0 && count >= maxResults { + if i < len(resp.Segment.BlobPrefixes)-1 || len(resp.Segment.BlobItems) > 0 || pager.More() { + pageInfo.HasNextPage = true + pageInfo.Cursor = &object.Name + } + + break L + } + } + + for i, item := range resp.Segment.BlobItems { + if item.Name == nil || (predicate != nil && !predicate(*item.Name)) { + continue + } + + object := serializeObjectInfo(item) + objects = append(objects, object) + count++ + + if maxResults > 0 && count >= maxResults { + if i < len(resp.Segment.BlobItems)-1 || pager.More() { + pageInfo.HasNextPage = true + pageInfo.Cursor = &object.Name + } + + break L + } } } diff --git a/connector/storage/azblob/utils.go b/connector/storage/azblob/utils.go index 6e920a9..55cc6a6 100644 --- a/connector/storage/azblob/utils.go +++ b/connector/storage/azblob/utils.go @@ -187,3 +187,19 @@ func serializeErrorResponse(err error) *schema.ConnectorError { return serializeAzureErrorResponse(respErr) } + +func makeListBlobsInclude(opts common.StorageObjectIncludeOptions) container.ListBlobsInclude { + return container.ListBlobsInclude{ + Versions: opts.Versions, + Metadata: opts.Metadata, + Tags: opts.Tags, + Copy: opts.Copy, + Snapshots: opts.Snapshots, + LegalHold: opts.LegalHold, + ImmutabilityPolicy: opts.Retention, + Permissions: opts.Permissions, + Deleted: false, + DeletedWithVersions: false, + UncommittedBlobs: false, + } +} diff --git a/connector/storage/bucket.go b/connector/storage/bucket.go index 714b9be..7de9f6f 100644 --- a/connector/storage/bucket.go +++ b/connector/storage/bucket.go @@ -3,7 +3,6 @@ package storage import ( "context" - "github.com/hasura/ndc-sdk-go/schema" "github.com/hasura/ndc-storage/connector/storage/common" ) @@ -37,10 +36,21 @@ func (m *Manager) UpdateBucket(ctx context.Context, args *common.UpdateBucketArg func (m *Manager) ListBuckets(ctx context.Context, clientID *common.StorageClientID, options *common.ListStorageBucketsOptions, predicate func(string) bool) (*common.StorageBucketListResults, error) { client, ok := m.GetClient(clientID) if !ok { - return nil, schema.InternalServerError("client not found", nil) + return &common.StorageBucketListResults{ + Buckets: []common.StorageBucket{}, + }, nil } - return client.ListBuckets(ctx, options, predicate) + results, err := client.ListBuckets(ctx, options, predicate) + if err != nil { + return nil, err + } + + for i := range results.Buckets { + results.Buckets[i].ClientID = string(client.id) + } + + return results, nil } // GetBucket gets bucket by name. @@ -50,7 +60,14 @@ func (m *Manager) GetBucket(ctx context.Context, bucketInfo *common.StorageBucke return nil, err } - return client.GetBucket(ctx, bucketName, options) + result, err := client.GetBucket(ctx, bucketName, options) + if err != nil { + return nil, err + } + + result.ClientID = string(client.id) + + return result, nil } // BucketExists checks if a bucket exists. diff --git a/connector/storage/common/arguments.go b/connector/storage/common/arguments.go index feb63e6..5d23f5b 100644 --- a/connector/storage/common/arguments.go +++ b/connector/storage/common/arguments.go @@ -10,7 +10,7 @@ import ( // ListStorageBucketArguments represent the input arguments for the ListBuckets methods. type ListStorageBucketArguments struct { // The maximum number of objects requested per batch. - MaxResults int `json:"maxResults,omitempty"` + MaxResults *int `json:"maxResults"` // StartAfter start listing lexically at this object onwards. StartAfter string `json:"startAfter,omitempty"` Where schema.Expression `json:"where" ndc:"predicate=StorageBucketFilter"` @@ -143,7 +143,13 @@ type StorageObjectIncludeOptions struct { Permissions bool Lifecycle bool Encryption bool - ObjectLock bool +} + +// IsEmpty checks if all include options are empty +func (soi StorageObjectIncludeOptions) IsEmpty() bool { + return !soi.Checksum && !soi.Tags && !soi.Versions && !soi.Metadata && + !soi.Copy && !soi.Snapshots && !soi.LegalHold && !soi.Retention && !soi.Permissions && + !soi.Lifecycle && !soi.Encryption } // ListStorageObjectsOptions holds all options of a list object request. diff --git a/connector/storage/common/storage.go b/connector/storage/common/storage.go index f57357a..9ffd6cd 100644 --- a/connector/storage/common/storage.go +++ b/connector/storage/common/storage.go @@ -66,16 +66,16 @@ type StorageClient interface { //nolint:interfacebloat // ListStorageBucketsOptions holds all options of a list bucket request. type ListStorageBucketsOptions struct { // Only list objects with the prefix - Prefix string `json:"prefix"` + Prefix string // The maximum number of objects requested per // batch, advanced use-case not useful for most // applications - MaxResults int `json:"maxResults"` + MaxResults *int // StartAfter start listing lexically at this object onwards. - StartAfter string `json:"startAfter"` + StartAfter string // Options to be included for the object information. - Include BucketIncludeOptions `json:"-"` - NumThreads int `json:"-"` + Include BucketIncludeOptions + NumThreads int } // BucketIncludeOptions contain include options for getting bucket information. @@ -87,11 +87,6 @@ type BucketIncludeOptions struct { ObjectLock bool } -// IsEmpty checks if all include options are false -func (bio BucketIncludeOptions) IsEmpty() bool { - return !bio.Tags && !bio.Versioning && !bio.Lifecycle && !bio.Encryption && !bio.ObjectLock -} - // BucketOptions hold options to get bucket information. type BucketOptions struct { Prefix string `json:"prefix"` @@ -107,6 +102,8 @@ type StorageBucketListResults struct { // StorageBucket the container for bucket metadata. type StorageBucket struct { + // Client ID + ClientID string `json:"clientId"` // The name of the bucket. Name string `json:"name"` // Bucket tags or metadata. @@ -613,11 +610,6 @@ type ObjectLifecycleTransition struct { Days *int `json:"days"` } -// IsEmpty checks if all properties of the object are empty. -func (fe ObjectLifecycleTransition) IsEmpty() bool { - return fe.StorageClass == nil && fe.Date == nil && fe.Days == nil -} - // LifecycleDelMarkerExpiration represents DelMarkerExpiration actions element in an ILM policy type ObjectLifecycleDelMarkerExpiration struct { Days *int `json:"days"` diff --git a/connector/storage/common/storage_test.go b/connector/storage/common/storage_test.go new file mode 100644 index 0000000..fa2b161 --- /dev/null +++ b/connector/storage/common/storage_test.go @@ -0,0 +1,11 @@ +package common + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func TestServerSideEncryptionConfiguration(t *testing.T) { + assert.Assert(t, ServerSideEncryptionConfiguration{}.IsEmpty()) +} diff --git a/connector/storage/common/transport.go b/connector/storage/common/transport.go index dc2d734..06adbed 100644 --- a/connector/storage/common/transport.go +++ b/connector/storage/common/transport.go @@ -92,7 +92,7 @@ func (mrt debugRoundTripper) RoundTrip(req *http.Request) (*http.Response, error if err != nil { span.SetStatus(codes.Error, err.Error()) span.RecordError(err) - slog.Debug("failed to execute the request: %s"+err.Error(), logAttrs...) + slog.Debug("failed to execute the request: "+err.Error(), logAttrs...) return resp, err } diff --git a/connector/storage/common/types.generated.go b/connector/storage/common/types.generated.go index a6758aa..62149fa 100644 --- a/connector/storage/common/types.generated.go +++ b/connector/storage/common/types.generated.go @@ -85,7 +85,7 @@ func (j *ListIncompleteUploadsOptions) FromValue(input map[string]any) error { // FromValue decodes values from map func (j *ListStorageBucketArguments) FromValue(input map[string]any) error { var err error - j.MaxResults, err = utils.GetIntDefault[int](input, "maxResults") + j.MaxResults, err = utils.GetNullableInt[int](input, "maxResults") if err != nil { return err } @@ -540,6 +540,7 @@ func (j StorageBucket) ToMap() map[string]any { if j.Autoclass != nil { r["autoclass"] = (*j.Autoclass) } + r["clientId"] = j.ClientID j_CORS := make([]any, len(j.CORS)) for i, j_CORS_v := range j.CORS { j_CORS[i] = j_CORS_v diff --git a/connector/storage/config.go b/connector/storage/config.go index ce34417..e1526ef 100644 --- a/connector/storage/config.go +++ b/connector/storage/config.go @@ -96,7 +96,7 @@ func (cc ClientConfig) Validate() error { return errors.New("unsupported storage client: " + string(baseConfig.Type)) } -func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger, version string) (*common.BaseClientConfig, common.StorageClient, error) { +func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger) (*common.BaseClientConfig, common.StorageClient, error) { if len(cc) == 0 { return nil, nil, errConfigEmpty } @@ -133,7 +133,7 @@ func (cc ClientConfig) toStorageClient(ctx context.Context, logger *slog.Logger, return nil, nil, err } - client, err := gcs.New(ctx, &gcsConfig, logger, version) + client, err := gcs.New(ctx, &gcsConfig, logger) return &baseConfig, client, err case common.AzureBlobStore: diff --git a/connector/storage/gcs/bucket.go b/connector/storage/gcs/bucket.go index 29f346d..e95086e 100644 --- a/connector/storage/gcs/bucket.go +++ b/connector/storage/gcs/bucket.go @@ -3,6 +3,7 @@ package gcs import ( "context" "errors" + "strings" "cloud.google.com/go/storage" "github.com/hasura/ndc-sdk-go/schema" @@ -48,11 +49,16 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc pager := c.client.Buckets(ctx, c.projectID) pager.Prefix = options.Prefix - var count int + var count, maxResults int var results []common.StorageBucket - maxResults := options.MaxResults pageInfo := common.StoragePaginationInfo{} + if options.MaxResults != nil { + maxResults = *options.MaxResults + } + + started := options.StartAfter == "" + for { bucket, err := pager.Next() if err != nil { @@ -66,6 +72,12 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc return nil, serializeErrorResponse(err) } + if !started { + started = options.StartAfter == bucket.Name + + continue + } + var cursor *string pi := pager.PageInfo() @@ -73,16 +85,22 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc cursor = &pi.Token } - if predicate == nil || predicate(bucket.Name) { - result := serializeBucketInfo(bucket) - results = append(results, result) - count++ + if (options.Prefix != "" && !strings.HasPrefix(bucket.Name, options.Prefix)) || (predicate != nil && !predicate(bucket.Name)) { + continue } + result := serializeBucketInfo(bucket) + results = append(results, result) + count++ + if maxResults > 0 && count >= maxResults { pageInfo.HasNextPage = pi.Remaining() > 0 pageInfo.Cursor = cursor + if pageInfo.Cursor == nil { + pageInfo.Cursor = &bucket.Name + } + break } } diff --git a/connector/storage/gcs/client.go b/connector/storage/gcs/client.go index 3b1a6bc..eb047b8 100644 --- a/connector/storage/gcs/client.go +++ b/connector/storage/gcs/client.go @@ -27,7 +27,7 @@ type Client struct { var _ common.StorageClient = &Client{} // New creates a new Minio client. -func New(ctx context.Context, config *ClientConfig, logger *slog.Logger, version string) (*Client, error) { +func New(ctx context.Context, config *ClientConfig, logger *slog.Logger) (*Client, error) { publicHost, err := config.ValidatePublicHost() if err != nil { return nil, err @@ -42,7 +42,7 @@ func New(ctx context.Context, config *ClientConfig, logger *slog.Logger, version return nil, errRequireProjectID } - opts, err := config.toClientOptions(ctx, logger, version) + opts, err := config.toClientOptions(ctx, logger) if err != nil { return nil, err } diff --git a/connector/storage/gcs/config.go b/connector/storage/gcs/config.go index e03f0d9..5baa6fd 100644 --- a/connector/storage/gcs/config.go +++ b/connector/storage/gcs/config.go @@ -61,10 +61,9 @@ type OtherConfig struct { Authentication AuthCredentials `json:"authentication" mapstructure:"authentication" yaml:"authentication"` } -func (cc ClientConfig) toClientOptions(ctx context.Context, logger *slog.Logger, version string) ([]option.ClientOption, error) { +func (cc ClientConfig) toClientOptions(ctx context.Context, logger *slog.Logger) ([]option.ClientOption, error) { opts := []option.ClientOption{ option.WithLogger(logger), - option.WithUserAgent(fmt.Sprintf("hasura/ndc-storage (%s)", version)), } cred, err := cc.Authentication.toCredentials() diff --git a/connector/storage/gcs/object.go b/connector/storage/gcs/object.go index dd4d8fa..100a04c 100644 --- a/connector/storage/gcs/object.go +++ b/connector/storage/gcs/object.go @@ -34,6 +34,7 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo q := c.validateListObjectsOptions(span, opts, false) pager := c.client.Bucket(bucketName).Objects(ctx, q) pageInfo := common.StoragePaginationInfo{} + started := opts.StartAfter == "" for { object, err := pager.Next() @@ -48,6 +49,12 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo return nil, serializeErrorResponse(err) } + if !started { + started = object.Name == opts.StartAfter + + continue + } + var cursor *string pi := pager.PageInfo() @@ -59,13 +66,19 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts *commo result := serializeObjectInfo(object) objects = append(objects, result) count++ - } - if maxResults > 0 && count >= maxResults { - pageInfo.HasNextPage = pi.Remaining() > 0 - pageInfo.Cursor = cursor + if maxResults > 0 && count >= maxResults { + if pi.Remaining() > 0 { + pageInfo.HasNextPage = true + pageInfo.Cursor = cursor - break + if pageInfo.Cursor == nil { + pageInfo.Cursor = &result.Name + } + } + + break + } } } diff --git a/connector/storage/gcs/utils.go b/connector/storage/gcs/utils.go index 1e5b220..c51e9ae 100644 --- a/connector/storage/gcs/utils.go +++ b/connector/storage/gcs/utils.go @@ -143,7 +143,7 @@ func serializeRetentionPolicy(retentionPolicy *storage.RetentionPolicy) *common. } } -func serializeObjectInfo(obj *storage.ObjectAttrs) common.StorageObject { +func serializeObjectInfo(obj *storage.ObjectAttrs) common.StorageObject { //nolint:cyclop object := common.StorageObject{ Bucket: obj.Bucket, Name: obj.Name, @@ -151,10 +151,17 @@ func serializeObjectInfo(obj *storage.ObjectAttrs) common.StorageObject { LastModified: obj.Updated, Size: &obj.Size, Metadata: obj.Metadata, - StorageClass: &obj.StorageClass, LegalHold: &obj.TemporaryHold, } + if obj.Name == "" { + object.Name = obj.Prefix + } + + if obj.StorageClass != "" { + object.StorageClass = &obj.StorageClass + } + if obj.Etag != "" { object.ETag = &obj.Etag } @@ -257,12 +264,18 @@ func (c *Client) validateListObjectsOptions(span trace.Span, opts *common.ListSt span.SetAttributes(attribute.Int("storage.options.max_results", opts.MaxResults)) } - return &storage.Query{ + result := &storage.Query{ Versions: opts.Include.Versions, Prefix: opts.Prefix, StartOffset: opts.StartAfter, SoftDeleted: includeDeleted, } + + if !opts.Recursive { + result.Delimiter = "/" + } + + return result } func serializeUploadObjectInfo(obj *storage.Writer) common.StorageUploadInfo { diff --git a/connector/storage/manager.go b/connector/storage/manager.go index 165e6bd..5e006ca 100644 --- a/connector/storage/manager.go +++ b/connector/storage/manager.go @@ -19,7 +19,7 @@ type Manager struct { } // NewManager creates a storage client manager instance. -func NewManager(ctx context.Context, configs []ClientConfig, logger *slog.Logger, version string) (*Manager, error) { +func NewManager(ctx context.Context, configs []ClientConfig, logger *slog.Logger) (*Manager, error) { if len(configs) == 0 { return nil, errors.New("failed to initialize storage clients: config is empty") } @@ -29,7 +29,7 @@ func NewManager(ctx context.Context, configs []ClientConfig, logger *slog.Logger } for i, config := range configs { - baseConfig, client, err := config.toStorageClient(ctx, logger, version) + baseConfig, client, err := config.toStorageClient(ctx, logger) if err != nil { return nil, fmt.Errorf("failed to initialize storage client %d: %w", i, err) } diff --git a/connector/storage/minio/bucket.go b/connector/storage/minio/bucket.go index 388fc8a..5029b15 100644 --- a/connector/storage/minio/bucket.go +++ b/connector/storage/minio/bucket.go @@ -58,21 +58,7 @@ func (mc *Client) ListBuckets(ctx context.Context, options *common.ListStorageBu return nil, serializeErrorResponse(err) } - var filteredBuckets []minio.BucketInfo - - if options.Prefix == "" && predicate == nil { - filteredBuckets = bucketInfos - } else { - for _, info := range bucketInfos { - if (options.Prefix != "" && !strings.HasPrefix(info.Name, options.Prefix)) || - (predicate != nil && !predicate(info.Name)) { - continue - } - - filteredBuckets = append(filteredBuckets, info) - } - } - + filteredBuckets, pageInfo := filterBuckets(bucketInfos, options, predicate) span.SetAttributes(attribute.Int("storage.bucket_count", len(bucketInfos))) if len(bucketInfos) == 0 { @@ -100,7 +86,8 @@ func (mc *Client) ListBuckets(ctx context.Context, options *common.ListStorageBu } return &common.StorageBucketListResults{ - Buckets: results, + Buckets: results, + PageInfo: pageInfo, }, nil } @@ -135,7 +122,8 @@ func (mc *Client) ListBuckets(ctx context.Context, options *common.ListStorageBu } return &common.StorageBucketListResults{ - Buckets: results, + Buckets: results, + PageInfo: pageInfo, }, nil } @@ -790,3 +778,43 @@ func (mc *Client) populateBucket(ctx context.Context, item minio.BucketInfo, opt return bucket, nil } + +func filterBuckets(bucketInfos []minio.BucketInfo, options *common.ListStorageBucketsOptions, predicate func(string) bool) ([]minio.BucketInfo, common.StoragePaginationInfo) { + pageInfo := common.StoragePaginationInfo{} + + if len(bucketInfos) == 0 || (options.Prefix == "" && predicate == nil && options.MaxResults == nil && options.StartAfter == "") { + return bucketInfos, pageInfo + } + + var count int + filteredBuckets := make([]minio.BucketInfo, 0) + started := options.StartAfter == "" + bucketLength := len(bucketInfos) + + for i, info := range bucketInfos { + if !started { + started = options.StartAfter == info.Name + + continue + } + + if (options.Prefix != "" && !strings.HasPrefix(info.Name, options.Prefix)) || + (predicate != nil && !predicate(info.Name)) { + continue + } + + filteredBuckets = append(filteredBuckets, info) + count++ + + if options.MaxResults != nil && count >= *options.MaxResults { + if i < bucketLength-1 { + pageInfo.HasNextPage = true + pageInfo.Cursor = &info.Name + } + + break + } + } + + return filteredBuckets, pageInfo +} diff --git a/connector/storage/minio/object.go b/connector/storage/minio/object.go index e56f9ed..7ffdc41 100644 --- a/connector/storage/minio/object.go +++ b/connector/storage/minio/object.go @@ -21,7 +21,7 @@ import ( ) // ListObjects list objects in a bucket. -func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { +func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { //nolint:funlen ctx, span := mc.startOtelSpan(ctx, "ListObjects", bucketName) defer span.End() @@ -33,9 +33,8 @@ func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *comm opts.MaxResults = 0 } - var count int objChan := mc.client.ListObjects(ctx, bucketName, mc.validateListObjectsOptions(span, opts)) - objects := make([]common.StorageObject, 0) + minioObjects := []minio.ObjectInfo{} for obj := range objChan { if obj.Err != nil { @@ -49,28 +48,48 @@ func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *comm continue } - if predicate == nil || (maxResults > 0 && count < maxResults) { - object := serializeObjectInfo(obj, true) - object.Bucket = bucketName + minioObjects = append(minioObjects, obj) + } - objects = append(objects, object) - count++ - } + if len(minioObjects) == 0 { + span.SetAttributes(attribute.Int("storage.object_count", 0)) + + return &common.StorageObjectListResults{ + Objects: []common.StorageObject{}, + }, nil } - span.SetAttributes(attribute.Int("storage.object_count", count)) + maxLength := len(minioObjects) + pageInfo := common.StoragePaginationInfo{} + + if maxResults > 0 && maxResults < maxLength { + maxLength = maxResults + pageInfo.HasNextPage = true + pageInfo.Cursor = &minioObjects[maxLength-1].Key + } + + objects := make([]common.StorageObject, maxLength) + + for i := range maxLength { + object := serializeObjectInfo(&minioObjects[i], true) + object.Bucket = bucketName + + objects[i] = object + } - if len(objects) == 0 || !opts.Include.LegalHold { + span.SetAttributes(attribute.Int("storage.object_count", maxLength)) + + if opts.Include.IsEmpty() { return &common.StorageObjectListResults{ - Objects: objects, + Objects: objects, + PageInfo: pageInfo, }, nil } if opts.NumThreads <= 1 { for i, object := range objects { - lhStatus, err := mc.GetObjectLegalHold(ctx, bucketName, object.Name, object.VersionID) + err := mc.populateObject(ctx, &object, opts.Include) if err == nil { - object.LegalHold = &lhStatus objects[i] = object } } @@ -87,13 +106,11 @@ func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *comm lhFunc := func(obj common.StorageObject, index int) { eg.Go(func() error { - lhStatus, err := mc.GetObjectLegalHold(ctx, bucketName, obj.Name, obj.VersionID) + err := mc.populateObject(ctx, &obj, opts.Include) if err == nil { - obj.LegalHold = &lhStatus + results[index] = obj } - results[index] = obj - return nil }) } @@ -103,12 +120,13 @@ func (mc *Client) ListObjects(ctx context.Context, bucketName string, opts *comm } if err := eg.Wait(); err != nil { - logger.Error("failed to fetch legal holds for objects: " + err.Error()) + logger.Error("failed to include object data: " + err.Error()) span.AddEvent("fetch_legal_holds_error", trace.WithAttributes(attribute.String("error", err.Error()))) } return &common.StorageObjectListResults{ - Objects: objects, + Objects: objects, + PageInfo: pageInfo, }, nil } @@ -350,28 +368,12 @@ func (mc *Client) StatObject(ctx context.Context, bucketName, objectName string, return nil, serializeErrorResponse(err) } - result := serializeObjectInfo(object, false) + result := serializeObjectInfo(&object, false) result.Bucket = bucketName - if opts.Include.Tags { - userTags, err := mc.GetObjectTags(ctx, bucketName, objectName, opts.VersionID) - if err != nil { - return nil, err - } - - result.Tags = userTags - } - - if opts.Include.LegalHold { - var versionID *string - if object.VersionID != "" { - versionID = &object.VersionID - } - - lhStatus, err := mc.GetObjectLegalHold(ctx, bucketName, object.Key, versionID) - if err == nil { - result.LegalHold = &lhStatus - } + err = mc.populateObject(ctx, &result, opts.Include) + if err != nil { + return nil, err } common.SetObjectInfoSpanAttributes(span, &result) @@ -840,3 +842,23 @@ func (mc *Client) GetObjectLockConfig(ctx context.Context, bucketName string) (* return result, nil } + +func (mc *Client) populateObject(ctx context.Context, result *common.StorageObject, include common.StorageObjectIncludeOptions) error { + if include.Tags { + userTags, err := mc.GetObjectTags(ctx, result.Bucket, result.Name, result.VersionID) + if err != nil { + return err + } + + result.Tags = userTags + } + + if include.LegalHold { + lhStatus, err := mc.GetObjectLegalHold(ctx, result.Bucket, result.Name, result.VersionID) + if err == nil { + result.LegalHold = &lhStatus + } + } + + return nil +} diff --git a/connector/storage/minio/utils.go b/connector/storage/minio/utils.go index 42efbb5..aaabf48 100644 --- a/connector/storage/minio/utils.go +++ b/connector/storage/minio/utils.go @@ -43,7 +43,7 @@ func serializeGrant(grant minio.Grant) common.StorageGrant { return g } -func serializeObjectInfo(obj minio.ObjectInfo, fromList bool) common.StorageObject { //nolint:funlen,gocognit,gocyclo,cyclop +func serializeObjectInfo(obj *minio.ObjectInfo, fromList bool) common.StorageObject { //nolint:funlen,gocognit,gocyclo,cyclop grants := make([]common.StorageGrant, len(obj.Grant)) for i, grant := range obj.Grant { diff --git a/connector/storage/object.go b/connector/storage/object.go index 78ad559..6a0ec5e 100644 --- a/connector/storage/object.go +++ b/connector/storage/object.go @@ -16,7 +16,9 @@ import ( func (m *Manager) ListObjects(ctx context.Context, bucketInfo common.StorageBucketArguments, opts *common.ListStorageObjectsOptions, predicate func(string) bool) (*common.StorageObjectListResults, error) { client, bucketName, err := m.GetClientAndBucket(bucketInfo.ClientID, bucketInfo.Bucket) if err != nil { - return nil, err + return &common.StorageObjectListResults{ //nolint:nilerr + Objects: []common.StorageObject{}, + }, nil } results, err := client.ListObjects(ctx, bucketName, opts, predicate) diff --git a/connector/testdata/02-get/query/storageObjects-gcs/expected.json b/connector/testdata/02-get/query/storageObjects-gcs/expected.json index c94423e..1e68174 100644 --- a/connector/testdata/02-get/query/storageObjects-gcs/expected.json +++ b/connector/testdata/02-get/query/storageObjects-gcs/expected.json @@ -75,7 +75,7 @@ } ], "pageInfo": { - "cursor": null, + "cursor": "public/hello.txt", "hasNextPage": true } } diff --git a/connector/testdata/02-get/query/storageObjects/request.json b/connector/testdata/02-get/query/storageObjects/request.json index 9adb09c..359a500 100644 --- a/connector/testdata/02-get/query/storageObjects/request.json +++ b/connector/testdata/02-get/query/storageObjects/request.json @@ -360,7 +360,6 @@ }, "arguments": { "recursive": { "type": "literal", "value": true }, - "maxResults": { "type": "literal", "value": 1 }, "where": { "type": "literal", "value": { diff --git a/connector/testdata/bucket/query/limit-3-6/expected.json b/connector/testdata/bucket/query/limit-3-6/expected.json new file mode 100644 index 0000000..c797e73 --- /dev/null +++ b/connector/testdata/bucket/query/limit-3-6/expected.json @@ -0,0 +1,89 @@ +[ + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "minio", + "name": "dummy-bucket-3", + "storageClass": null, + "tags": null + }, + { + "clientId": "minio", + "name": "dummy-bucket-4", + "storageClass": null, + "tags": null + }, + { + "clientId": "minio", + "name": "dummy-bucket-5", + "storageClass": null, + "tags": null + } + ], + "pageInfo": { "cursor": "dummy-bucket-5", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "azblob", + "name": "dummy-bucket-3", + "storageClass": null, + "tags": null + }, + { + "clientId": "azblob", + "name": "dummy-bucket-4", + "storageClass": null, + "tags": null + }, + { + "clientId": "azblob", + "name": "dummy-bucket-5", + "storageClass": null, + "tags": null + } + ], + "pageInfo": { "cursor": "dummy-bucket-5", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "gcs", + "name": "dummy-bucket-3", + "storageClass": "STANDARD", + "tags": null + }, + { + "clientId": "gcs", + "name": "dummy-bucket-4", + "storageClass": "STANDARD", + "tags": null + }, + { + "clientId": "gcs", + "name": "dummy-bucket-5", + "storageClass": "STANDARD", + "tags": null + } + ], + "pageInfo": { "cursor": "dummy-bucket-5", "hasNextPage": true } + } + } + ] + } +] diff --git a/connector/testdata/bucket/query/limit-3-6/request.json b/connector/testdata/bucket/query/limit-3-6/request.json new file mode 100644 index 0000000..5e94437 --- /dev/null +++ b/connector/testdata/bucket/query/limit-3-6/request.json @@ -0,0 +1,97 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 3 + }, + "startAfter": { + "type": "literal", + "value": "dummy-bucket-2" + }, + "where": { + "type": "literal", + "value": { + "expressions": [ + { + "column": { "type": "column", "name": "bucket" }, + "operator": "_starts_with", + "type": "binary_comparison_operator", + "value": { "type": "scalar", "value": "dummy" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + } + ], + "type": "and" + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection": "storageBuckets", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/bucket/query/limit-3/expected.json b/connector/testdata/bucket/query/limit-3/expected.json new file mode 100644 index 0000000..d79d5bb --- /dev/null +++ b/connector/testdata/bucket/query/limit-3/expected.json @@ -0,0 +1,89 @@ +[ + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "minio", + "name": "dummy-bucket-0", + "storageClass": null, + "tags": null + }, + { + "clientId": "minio", + "name": "dummy-bucket-1", + "storageClass": null, + "tags": null + }, + { + "clientId": "minio", + "name": "dummy-bucket-2", + "storageClass": null, + "tags": null + } + ], + "pageInfo": { "cursor": "dummy-bucket-2", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "azblob", + "name": "dummy-bucket-0", + "storageClass": null, + "tags": null + }, + { + "clientId": "azblob", + "name": "dummy-bucket-1", + "storageClass": null, + "tags": null + }, + { + "clientId": "azblob", + "name": "dummy-bucket-2", + "storageClass": null, + "tags": null + } + ], + "pageInfo": { "cursor": "dummy-bucket-2", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "gcs", + "name": "dummy-bucket-0", + "storageClass": "STANDARD", + "tags": null + }, + { + "clientId": "gcs", + "name": "dummy-bucket-1", + "storageClass": "STANDARD", + "tags": null + }, + { + "clientId": "gcs", + "name": "dummy-bucket-2", + "storageClass": "STANDARD", + "tags": null + } + ], + "pageInfo": { "cursor": "dummy-bucket-2", "hasNextPage": true } + } + } + ] + } +] diff --git a/connector/testdata/bucket/query/limit-3/request.json b/connector/testdata/bucket/query/limit-3/request.json new file mode 100644 index 0000000..a6cee05 --- /dev/null +++ b/connector/testdata/bucket/query/limit-3/request.json @@ -0,0 +1,97 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 3 + }, + "startAfter": { + "type": "literal", + "value": null + }, + "where": { + "type": "literal", + "value": { + "expressions": [ + { + "column": { "type": "column", "name": "bucket" }, + "operator": "_starts_with", + "type": "binary_comparison_operator", + "value": { "type": "scalar", "value": "dummy" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + } + ], + "type": "and" + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection": "storageBuckets", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/bucket/query/limit-9-12/expected.json b/connector/testdata/bucket/query/limit-9-12/expected.json new file mode 100644 index 0000000..105b6f3 --- /dev/null +++ b/connector/testdata/bucket/query/limit-9-12/expected.json @@ -0,0 +1,53 @@ +[ + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "minio", + "name": "dummy-bucket-9", + "storageClass": null, + "tags": null + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "azblob", + "name": "dummy-bucket-9", + "storageClass": null, + "tags": null + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [ + { + "clientId": "gcs", + "name": "dummy-bucket-9", + "storageClass": "STANDARD", + "tags": null + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/bucket/query/limit-9-12/request.json b/connector/testdata/bucket/query/limit-9-12/request.json new file mode 100644 index 0000000..7d962da --- /dev/null +++ b/connector/testdata/bucket/query/limit-9-12/request.json @@ -0,0 +1,97 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 3 + }, + "startAfter": { + "type": "literal", + "value": "dummy-bucket-8" + }, + "where": { + "type": "literal", + "value": { + "expressions": [ + { + "column": { "type": "column", "name": "bucket" }, + "operator": "_starts_with", + "type": "binary_comparison_operator", + "value": { "type": "scalar", "value": "dummy" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + } + ], + "type": "and" + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection": "storageBuckets", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/bucket/query/no-result-2/expected.json b/connector/testdata/bucket/query/no-result-2/expected.json new file mode 100644 index 0000000..e40380d --- /dev/null +++ b/connector/testdata/bucket/query/no-result-2/expected.json @@ -0,0 +1,12 @@ +[ + { + "rows": [ + { + "__value": { + "buckets": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/bucket/query/no-result-2/request.json b/connector/testdata/bucket/query/no-result-2/request.json new file mode 100644 index 0000000..efa7bfb --- /dev/null +++ b/connector/testdata/bucket/query/no-result-2/request.json @@ -0,0 +1,89 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 3 + }, + "startAfter": { + "type": "literal", + "value": null + }, + "where": { + "type": "literal", + "value": { + "expressions": [ + { + "column": { "type": "column", "name": "bucket" }, + "operator": "_starts_with", + "type": "binary_comparison_operator", + "value": { "type": "scalar", "value": "dummy" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + } + ], + "type": "and" + } + } + }, + "variables": [{ "$clientId": "not-found" }], + "collection": "storageBuckets", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "name": { + "column": "name", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/bucket/query/no-result/expected.json b/connector/testdata/bucket/query/no-result/expected.json new file mode 100644 index 0000000..1797773 --- /dev/null +++ b/connector/testdata/bucket/query/no-result/expected.json @@ -0,0 +1,32 @@ +[ + { + "rows": [ + { + "__value": { + "buckets": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "buckets": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/bucket/query/no-result/request.json b/connector/testdata/bucket/query/no-result/request.json new file mode 100644 index 0000000..e8624d5 --- /dev/null +++ b/connector/testdata/bucket/query/no-result/request.json @@ -0,0 +1,93 @@ +{ + "arguments": { + "maxResults": { + "type": "literal", + "value": 3 + }, + "startAfter": { + "type": "literal", + "value": null + }, + "where": { + "type": "literal", + "value": { + "expressions": [ + { + "column": { "type": "column", "name": "bucket" }, + "operator": "_starts_with", + "type": "binary_comparison_operator", + "value": { "type": "scalar", "value": "abcxyz" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + } + ], + "type": "and" + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection": "storageBuckets", + "collection_relationships": {}, + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "buckets": { + "column": "buckets", + "fields": { + "fields": { + "fields": { + "name": { + "column": "name", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + }, + "tags": { + "column": "tags", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + } +} diff --git a/connector/testdata/object/query/no-result-2/expected.json b/connector/testdata/object/query/no-result-2/expected.json new file mode 100644 index 0000000..c6f4f48 --- /dev/null +++ b/connector/testdata/object/query/no-result-2/expected.json @@ -0,0 +1,12 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/object/query/no-result-2/request.json b/connector/testdata/object/query/no-result-2/request.json new file mode 100644 index 0000000..7c5b8b7 --- /dev/null +++ b/connector/testdata/object/query/no-result-2/request.json @@ -0,0 +1,101 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 3 }, + "startAfter": { + "type": "literal", + "value": "movies/1990s/movies.json" + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies" } + } + ] + } + } + }, + "variables": [{ "$clientId": "not-found" }], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/no-result/expected.json b/connector/testdata/object/query/no-result/expected.json new file mode 100644 index 0000000..87dae45 --- /dev/null +++ b/connector/testdata/object/query/no-result/expected.json @@ -0,0 +1,32 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/object/query/no-result/request.json b/connector/testdata/object/query/no-result/request.json new file mode 100644 index 0000000..0cdc20d --- /dev/null +++ b/connector/testdata/object/query/no-result/request.json @@ -0,0 +1,105 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 3 }, + "startAfter": { + "type": "literal", + "value": "movies/1990s/movies.json" + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "not-found" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/non-recursive-file/expected.json b/connector/testdata/object/query/non-recursive-file/expected.json new file mode 100644 index 0000000..e3b690b --- /dev/null +++ b/connector/testdata/object/query/non-recursive-file/expected.json @@ -0,0 +1,56 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1920s/movies.json", + "size": 3463423, + "storageClass": "STANDARD" + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1920s/movies.json", + "size": 3463423, + "storageClass": "Hot" + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1920s/movies.json", + "size": 3463423, + "storageClass": "STANDARD" + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/object/query/non-recursive-file/request.json b/connector/testdata/object/query/non-recursive-file/request.json new file mode 100644 index 0000000..1122a30 --- /dev/null +++ b/connector/testdata/object/query/non-recursive-file/request.json @@ -0,0 +1,101 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": false }, + "maxResults": { "type": "literal", "value": 3 }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies/1920s/" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/non-recursive-folder-2/expected.json b/connector/testdata/object/query/non-recursive-folder-2/expected.json new file mode 100644 index 0000000..6e15551 --- /dev/null +++ b/connector/testdata/object/query/non-recursive-folder-2/expected.json @@ -0,0 +1,76 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1930s/", + "size": 0, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1940s/", + "size": 0, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1950s/", + "size": 0, + "storageClass": null + } + ], + "pageInfo": { "cursor": "movies/1950s/", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1920s/", + "size": null, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1930s/", + "size": null, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1940s/", + "size": null, + "storageClass": null + } + ], + "pageInfo": { "cursor": "movies/1940s/", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/object/query/non-recursive-folder-2/request.json b/connector/testdata/object/query/non-recursive-folder-2/request.json new file mode 100644 index 0000000..d148636 --- /dev/null +++ b/connector/testdata/object/query/non-recursive-folder-2/request.json @@ -0,0 +1,105 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": false }, + "maxResults": { "type": "literal", "value": 3 }, + "startAfter": { + "type": "literal", + "value": "movies/1920s/" + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies/" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/non-recursive-folder/expected.json b/connector/testdata/object/query/non-recursive-folder/expected.json new file mode 100644 index 0000000..363bf22 --- /dev/null +++ b/connector/testdata/object/query/non-recursive-folder/expected.json @@ -0,0 +1,98 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1900s/", + "size": 0, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1910s/", + "size": 0, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1920s/", + "size": 0, + "storageClass": null + } + ], + "pageInfo": { "cursor": "movies/1920s/", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1900s/", + "size": null, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1910s/", + "size": null, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1920s/", + "size": null, + "storageClass": null + } + ], + "pageInfo": { "cursor": "movies/1920s/", "hasNextPage": true } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1900s/", + "size": 0, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1910s/", + "size": 0, + "storageClass": null + }, + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1920s/", + "size": 0, + "storageClass": null + } + ], + "pageInfo": { "cursor": "movies/1920s/", "hasNextPage": true } + } + } + ] + } +] diff --git a/connector/testdata/object/query/non-recursive-folder/request.json b/connector/testdata/object/query/non-recursive-folder/request.json new file mode 100644 index 0000000..10ef069 --- /dev/null +++ b/connector/testdata/object/query/non-recursive-folder/request.json @@ -0,0 +1,101 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": false }, + "maxResults": { "type": "literal", "value": 3 }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies/" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/recursive-limit-3-6/expected.json b/connector/testdata/object/query/recursive-limit-3-6/expected.json new file mode 100644 index 0000000..ee2fc0e --- /dev/null +++ b/connector/testdata/object/query/recursive-limit-3-6/expected.json @@ -0,0 +1,107 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1930s/movies.json", + "size": 2921862, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1940s/movies.json", + "size": 2972875, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1950s/movies.json", + "size": 2123294, + "storageClass": "STANDARD" + } + ], + "pageInfo": { + "cursor": "movies/1950s/movies.json", + "hasNextPage": true + } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1930s/movies.json", + "size": 2921862, + "storageClass": "Hot" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1940s/movies.json", + "size": 2972875, + "storageClass": "Hot" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1950s/movies.json", + "size": 2123294, + "storageClass": "Hot" + } + ], + "pageInfo": { + "cursor": "movies/1950s/movies.json", + "hasNextPage": true + } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1930s/movies.json", + "size": 2921862, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1940s/movies.json", + "size": 2972875, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1950s/movies.json", + "size": 2123294, + "storageClass": "STANDARD" + } + ], + "pageInfo": { + "cursor": "movies/1950s/movies.json", + "hasNextPage": true + } + } + } + ] + } +] diff --git a/connector/testdata/object/query/recursive-limit-3-6/request.json b/connector/testdata/object/query/recursive-limit-3-6/request.json new file mode 100644 index 0000000..2109809 --- /dev/null +++ b/connector/testdata/object/query/recursive-limit-3-6/request.json @@ -0,0 +1,105 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 3 }, + "startAfter": { + "type": "literal", + "value": "movies/1920s/movies.json" + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/recursive-limit-3/expected.json b/connector/testdata/object/query/recursive-limit-3/expected.json new file mode 100644 index 0000000..da2611d --- /dev/null +++ b/connector/testdata/object/query/recursive-limit-3/expected.json @@ -0,0 +1,107 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1900s/movies.json", + "size": 100041, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1910s/movies.json", + "size": 2007693, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/1920s/movies.json", + "size": 3463423, + "storageClass": "STANDARD" + } + ], + "pageInfo": { + "cursor": "movies/1920s/movies.json", + "hasNextPage": true + } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1900s/movies.json", + "size": 100041, + "storageClass": "Hot" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1910s/movies.json", + "size": 2007693, + "storageClass": "Hot" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/1920s/movies.json", + "size": 3463423, + "storageClass": "Hot" + } + ], + "pageInfo": { + "cursor": "movies/1920s/movies.json", + "hasNextPage": true + } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1900s/movies.json", + "size": 100041, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1910s/movies.json", + "size": 2007693, + "storageClass": "STANDARD" + }, + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/1920s/movies.json", + "size": 3463423, + "storageClass": "STANDARD" + } + ], + "pageInfo": { + "cursor": "movies/1920s/movies.json", + "hasNextPage": true + } + } + } + ] + } +] diff --git a/connector/testdata/object/query/recursive-limit-3/request.json b/connector/testdata/object/query/recursive-limit-3/request.json new file mode 100644 index 0000000..dcf9ced --- /dev/null +++ b/connector/testdata/object/query/recursive-limit-3/request.json @@ -0,0 +1,101 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 3 }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/connector/testdata/object/query/recursive-limit-9-12/expected.json b/connector/testdata/object/query/recursive-limit-9-12/expected.json new file mode 100644 index 0000000..347650f --- /dev/null +++ b/connector/testdata/object/query/recursive-limit-9-12/expected.json @@ -0,0 +1,56 @@ +[ + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "minio", + "name": "movies/2000s/movies.json", + "size": 2472795, + "storageClass": "STANDARD" + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "azblob", + "name": "movies/2000s/movies.json", + "size": 2472795, + "storageClass": "Hot" + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + }, + { + "rows": [ + { + "__value": { + "objects": [ + { + "bucket": "dummy-bucket-0", + "clientId": "gcs", + "name": "movies/2000s/movies.json", + "size": 2472795, + "storageClass": "STANDARD" + } + ], + "pageInfo": { "cursor": null, "hasNextPage": false } + } + } + ] + } +] diff --git a/connector/testdata/object/query/recursive-limit-9-12/request.json b/connector/testdata/object/query/recursive-limit-9-12/request.json new file mode 100644 index 0000000..1254858 --- /dev/null +++ b/connector/testdata/object/query/recursive-limit-9-12/request.json @@ -0,0 +1,105 @@ +{ + "collection": "storageObjects", + "query": { + "fields": { + "__value": { + "column": "__value", + "fields": { + "fields": { + "objects": { + "column": "objects", + "fields": { + "fields": { + "fields": { + "clientId": { + "column": "clientId", + "type": "column" + }, + "bucket": { + "column": "bucket", + "type": "column" + }, + "name": { + "column": "name", + "type": "column" + }, + "size": { + "column": "size", + "type": "column" + }, + "storageClass": { + "column": "storageClass", + "type": "column" + } + }, + "type": "object" + }, + "type": "array" + }, + "type": "column" + }, + "pageInfo": { + "column": "pageInfo", + "fields": { + "fields": { + "cursor": { + "column": "cursor", + "type": "column" + }, + "hasNextPage": { + "column": "hasNextPage", + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + }, + "type": "object" + }, + "type": "column" + } + } + }, + "arguments": { + "recursive": { "type": "literal", "value": true }, + "maxResults": { "type": "literal", "value": 3 }, + "startAfter": { + "type": "literal", + "value": "movies/1990s/movies.json" + }, + "where": { + "type": "literal", + "value": { + "type": "and", + "expressions": [ + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "clientId", "path": [] }, + "operator": "_eq", + "value": { "type": "variable", "name": "$clientId" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "bucket", "path": [] }, + "operator": "_eq", + "value": { "type": "scalar", "value": "dummy-bucket-0" } + }, + { + "type": "binary_comparison_operator", + "column": { "type": "column", "name": "object", "path": [] }, + "operator": "_starts_with", + "value": { "type": "scalar", "value": "movies" } + } + ] + } + } + }, + "variables": [ + { "$clientId": "minio" }, + { "$clientId": "azblob" }, + { "$clientId": "gcs" } + ], + "collection_relationships": {} +} diff --git a/go.mod b/go.mod index 4030536..5ceabc4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 - github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 github.com/alecthomas/kong v1.6.1 github.com/go-viper/mapstructure/v2 v2.2.1 github.com/hasura/ndc-sdk-go v1.7.1-0.20250116172618-e81c1199f349 diff --git a/go.sum b/go.sum index b4bfcfd..d0cecd0 100644 --- a/go.sum +++ b/go.sum @@ -29,14 +29,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvUL github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo= -github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0 h1:RTTsXUJWn0jumeX62Mb153wYXykqnrzYBYDeHp0kiuk= -github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel v0.4.0/go.mod h1:k4MMjrPHIEK+umaMGk1GNLgjEybJZ9mHSRDZ+sDFv3Y= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= @@ -215,8 +211,6 @@ go.opentelemetry.io/contrib/propagators/b3 v1.29.0 h1:hNjyoRsAACnhoOLWupItUjABze go.opentelemetry.io/contrib/propagators/b3 v1.29.0/go.mod h1:E76MTitU1Niwo5NSN+mVxkyLu4h4h7Dp/yh38F2WuIU= go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/exporters/jaeger v1.16.0 h1:YhxxmXZ011C0aDZKoNw+juVWAmEfv/0W2XBOv9aHTaA= -go.opentelemetry.io/otel/exporters/jaeger v1.16.0/go.mod h1:grYbBo/5afWlPpdPZYhyn78Bk04hnvxn2+hvxQhKIQM= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.5.0 h1:iWyFL+atC9S1e6MFDLNUZieyKTmsrvsDzuozUDbFg8E= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.5.0/go.mod h1:0Ur7rPCJmkHksYcBywsFXnKBG3pqGl4TGltZ+T3qhSA= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.5.0 h1:4d++HQ+Ihdl+53zSjtsCUFDmNMju2FC9qFkUlTxPLqo= diff --git a/tests/configuration/configuration.yaml b/tests/configuration/configuration.yaml index f86dde0..b701bf6 100644 --- a/tests/configuration/configuration.yaml +++ b/tests/configuration/configuration.yaml @@ -47,6 +47,16 @@ clients: defaultPresignedExpiry: "1h" allowedBuckets: - azblob-bucket-test + - dummy-bucket-0 + - dummy-bucket-1 + - dummy-bucket-2 + - dummy-bucket-3 + - dummy-bucket-4 + - dummy-bucket-5 + - dummy-bucket-6 + - dummy-bucket-7 + - dummy-bucket-8 + - dummy-bucket-9 - id: azblob-connstr type: azblob defaultBucket: diff --git a/tests/engine/app/metadata/StorageBucket.hml b/tests/engine/app/metadata/StorageBucket.hml index 04a0c1b..5106a7a 100644 --- a/tests/engine/app/metadata/StorageBucket.hml +++ b/tests/engine/app/metadata/StorageBucket.hml @@ -722,6 +722,8 @@ definition: type: StorageBucketVersioningConfiguration - name: website type: BucketWebsite + - name: clientId + type: String! graphql: typeName: StorageBucket inputTypeName: StorageBucketInput @@ -760,6 +762,7 @@ definition: - tags - versioning - website + - clientId --- kind: Command diff --git a/tests/engine/app/metadata/storage.hml b/tests/engine/app/metadata/storage.hml index 35eccc7..a8fb0d0 100644 --- a/tests/engine/app/metadata/storage.hml +++ b/tests/engine/app/metadata/storage.hml @@ -889,6 +889,10 @@ definition: underlying_type: type: named name: BucketAutoclass + clientId: + type: + type: named + name: String cors: type: type: nullable