Skip to content

Commit

Permalink
add more unit tests
Browse files Browse the repository at this point in the history
  • Loading branch information
hgiasac committed Feb 1, 2025
1 parent adb4d67 commit 7a936fd
Show file tree
Hide file tree
Showing 60 changed files with 2,645 additions and 162 deletions.
3 changes: 1 addition & 2 deletions connector/connector.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/hasura/ndc-sdk-go/connector"
"github.com/hasura/ndc-sdk-go/schema"
"github.com/hasura/ndc-sdk-go/utils"
"github.com/hasura/ndc-storage/configuration/version"
"github.com/hasura/ndc-storage/connector/functions"
"github.com/hasura/ndc-storage/connector/storage"
"github.com/hasura/ndc-storage/connector/types"
Expand Down Expand Up @@ -73,7 +72,7 @@ func (c *Connector) ParseConfiguration(ctx context.Context, configurationDir str
func (c *Connector) TryInitState(ctx context.Context, configuration *types.Configuration, metrics *connector.TelemetryState) (*types.State, error) {
logger := connector.GetLogger(ctx)

manager, err := storage.NewManager(ctx, configuration.Clients, logger, version.BuildVersion)
manager, err := storage.NewManager(ctx, configuration.Clients, logger)
if err != nil {
return nil, err
}
Expand Down
18 changes: 11 additions & 7 deletions connector/connector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@ import (
)

func TestConnector(t *testing.T) {
setConnectorTestEnv(t)

for _, dir := range []string{"01-setup", "02-get", "03-cleanup"} {
ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{
Configuration: "../tests/configuration",
TestDataDir: filepath.Join("testdata", dir),
})
}
}

func setConnectorTestEnv(t *testing.T) {
azureBlobEndpoint := "http://local.hasura.dev:10000"
azureAccountName := "local"
azureAccountKey := "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
Expand All @@ -30,11 +41,4 @@ func TestConnector(t *testing.T) {
t.Setenv("GOOGLE_PROJECT_ID", "test-local-project")
t.Setenv("GOOGLE_STORAGE_ENDPOINT", "http://localhost:10010/storage/v1/")
t.Setenv("GOOGLE_STORAGE_CREDENTIALS_FILE", "../tests/certs/service_account.json")

for _, dir := range []string{"01-setup", "02-get", "03-cleanup"} {
ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{
Configuration: "../tests/configuration",
TestDataDir: filepath.Join("testdata", dir),
})
}
}
6 changes: 3 additions & 3 deletions connector/functions/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func ProcedureCreateStorageBucket(ctx context.Context, state *types.State, args

// FunctionStorageBuckets list all buckets.
func FunctionStorageBuckets(ctx context.Context, state *types.State, args *common.ListStorageBucketArguments) (common.StorageBucketListResults, error) {
if args.MaxResults <= 0 {
if args.MaxResults != nil && *args.MaxResults <= 0 {
return common.StorageBucketListResults{}, schema.UnprocessableContentError("maxResults must be larger than 0", nil)
}

Expand Down Expand Up @@ -54,7 +54,7 @@ func FunctionStorageBuckets(ctx context.Context, state *types.State, args *commo
Versioning: request.Include.Versions,
Lifecycle: request.Include.Lifecycle,
Encryption: request.Include.Encryption,
ObjectLock: request.Include.ObjectLock,
ObjectLock: request.IncludeObjectLock,
},
NumThreads: state.Concurrency.Query,
}, predicate)
Expand All @@ -79,7 +79,7 @@ func FunctionStorageBucket(ctx context.Context, state *types.State, args *common
Versioning: request.Include.Versions,
Lifecycle: request.Include.Lifecycle,
Encryption: request.Include.Encryption,
ObjectLock: request.Include.ObjectLock,
ObjectLock: request.IncludeObjectLock,
},
NumThreads: state.Concurrency.Query,
})
Expand Down
9 changes: 5 additions & 4 deletions connector/functions/internal/predicate.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ import (

// PredicateEvaluator the structured predicate result which is evaluated from the raw expression.
type PredicateEvaluator struct {
ClientID *common.StorageClientID
IsValid bool
Include common.StorageObjectIncludeOptions
ClientID *common.StorageClientID
IsValid bool
Include common.StorageObjectIncludeOptions
IncludeObjectLock bool

variables map[string]any
BucketPredicate StringFilterPredicate
Expand Down Expand Up @@ -137,7 +138,7 @@ func (pe *PredicateEvaluator) EvalSelection(selection schema.NestedField) error
}

if _, ok := expr.Fields["objectLock"]; ok {
pe.Include.ObjectLock = true
pe.IncludeObjectLock = true
}
}

Expand Down
131 changes: 131 additions & 0 deletions connector/query_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package connector

import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"path/filepath"
"testing"

"github.com/hasura/ndc-sdk-go/ndctest"
"github.com/hasura/ndc-sdk-go/schema"
"gotest.tools/v3/assert"
)

func TestConnectorQueries(t *testing.T) {
connectorHost := "http://localhost:8080"
clientIDs := []string{"minio", "azblob", "gcs"}

for _, cid := range clientIDs {
t.Run("create_bucket_"+cid, func(t *testing.T) {
procedureRequest := schema.MutationRequest{
CollectionRelationships: schema.MutationRequestCollectionRelationships{},
Operations: []schema.MutationOperation{},
}

for i := range 10 {
procedureRequest.Operations = append(procedureRequest.Operations, schema.MutationOperation{
Type: schema.MutationOperationProcedure,
Name: "createStorageBucket",
Arguments: []byte(fmt.Sprintf(`{
"clientId": "%s",
"name": "dummy-bucket-%d"
}`, cid, i)),
})
}

rawBody, err := json.Marshal(procedureRequest)
assert.NilError(t, err)

resp, err := http.DefaultClient.Post(connectorHost+"/mutation", "application/json", bytes.NewReader(rawBody))
assert.NilError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)
})
}

objectFixtures := map[string]string{
"movies/1900s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1900s.json",
"movies/1910s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1910s.json",
"movies/1920s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1920s.json",
"movies/1930s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1930s.json",
"movies/1940s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1940s.json",
"movies/1950s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1950s.json",
"movies/1960s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1960s.json",
"movies/1970s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1970s.json",
"movies/1980s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1980s.json",
"movies/1990s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-1990s.json",
"movies/2000s/movies.json": "https://raw.githubusercontent.com/prust/wikipedia-movie-data/refs/heads/master/movies-2000s.json",
}

for key, value := range objectFixtures {
resp, err := http.DefaultClient.Get(value)
assert.NilError(t, err)
assert.Equal(t, http.StatusOK, resp.StatusCode)

rawBody, err := io.ReadAll(resp.Body)
assert.NilError(t, err)
resp.Body.Close()

for _, cid := range clientIDs {
t.Run(fmt.Sprintf("upload_object_%s/%s", cid, key), func(t *testing.T) {
arguments := map[string]any{
"clientId": cid,
"bucket": "dummy-bucket-0",
"data": string(rawBody),
"object": key,
"options": map[string]any{
"cacheControl": "max-age=100",
"contentDisposition": "attachment",
"contentLanguage": "en-US",
"contentType": "application/json",
"expires": "2099-01-01T00:00:00Z",
"sendContentMd5": true,
"metadata": map[string]any{
"Foo": "Baz",
},
"tags": map[string]any{
"category": "movie",
},
},
}

rawArguments, err := json.Marshal(arguments)
assert.NilError(t, err)

procedureRequest := schema.MutationRequest{
CollectionRelationships: schema.MutationRequestCollectionRelationships{},
Operations: []schema.MutationOperation{
{
Type: schema.MutationOperationProcedure,
Name: "uploadStorageObjectText",
Arguments: rawArguments,
Fields: schema.NewNestedObject(map[string]schema.FieldEncoder{
"name": schema.NewColumnField("name", nil),
"size": schema.NewColumnField("size", nil),
}).Encode(),
},
},
}

uploadBytes, err := json.Marshal(procedureRequest)
assert.NilError(t, err)

resp, err = http.DefaultClient.Post(connectorHost+"/mutation", "application/json", bytes.NewReader(uploadBytes))
assert.NilError(t, err)
resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
})
}
}

setConnectorTestEnv(t)

for _, dir := range []string{"bucket", "object"} {
ndctest.TestConnector(t, &Connector{}, ndctest.TestConnectorOptions{
Configuration: "../tests/configuration",
TestDataDir: filepath.Join("testdata", dir),
})
}
}
3 changes: 3 additions & 0 deletions connector/schema.generated.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 10 additions & 10 deletions connector/storage/azblob/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,12 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc
opts.Prefix = &options.Prefix
}

maxResults := int32(options.MaxResults)
if options.MaxResults > 0 && predicate == nil {
var maxResults int32
if options.MaxResults != nil && *options.MaxResults > 0 && predicate == nil {
maxResults = int32(*options.MaxResults)
opts.MaxResults = &maxResults

span.SetAttributes(attribute.Int("storage.options.max_results", int(maxResults)))
}

if options.StartAfter != "" {
Expand All @@ -71,6 +74,7 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc
var results []common.StorageBucket
pageInfo := common.StoragePaginationInfo{}

L:
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
Expand All @@ -80,7 +84,7 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc
return nil, serializeErrorResponse(err)
}

for _, container := range resp.ContainerItems {
for i, container := range resp.ContainerItems {
if container.Name == nil || (predicate != nil && !predicate(*container.Name)) {
continue
}
Expand Down Expand Up @@ -136,16 +140,12 @@ func (c *Client) ListBuckets(ctx context.Context, options *common.ListStorageBuc
count++

if maxResults > 0 && count >= maxResults {
if pager.More() {
if i < len(resp.ContainerItems)-1 || pager.More() {
pageInfo.HasNextPage = true
pageInfo.Cursor = resp.NextMarker
}

if resp.Marker != nil && *resp.Marker != "" {
pageInfo.Cursor = resp.Marker
pageInfo.Cursor = &result.Name
}

break
break L
}
}
}
Expand Down
3 changes: 0 additions & 3 deletions connector/storage/azblob/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,9 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/tracing/azotel"
"github.com/hasura/ndc-sdk-go/utils"
"github.com/hasura/ndc-storage/connector/storage/common"
"github.com/invopop/jsonschema"
"go.opentelemetry.io/otel"
)

var (
Expand Down Expand Up @@ -66,7 +64,6 @@ func (cc ClientConfig) toAzureBlobClient(logger *slog.Logger) (*azblob.Client, e
IncludeBody: isDebug,
},
InsecureAllowCredentialWithHTTP: !useSSL,
TracingProvider: azotel.NewTracingProvider(otel.GetTracerProvider(), nil),
Transport: &http.Client{
Transport: transport,
},
Expand Down
Loading

0 comments on commit 7a936fd

Please sign in to comment.