From d4a33ada4764e075e9bda5a58d1af0167330d610 Mon Sep 17 00:00:00 2001 From: Tiago Peczenyj Date: Thu, 23 Aug 2018 11:00:08 +0200 Subject: [PATCH 1/3] fix issue #66 I am not sure if I can provide a unit test for this case, my problem is iterate over a huge s3 bucket without manually handle the next marker --- awss3/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awss3/store.go b/awss3/store.go index f405f1f..a89fb86 100644 --- a/awss3/store.go +++ b/awss3/store.go @@ -307,7 +307,7 @@ func (f *FS) List(ctx context.Context, q cloudstorage.Query) (*cloudstorage.Obje } if resp.IsTruncated != nil && *resp.IsTruncated { - q.Marker = *resp.Contents[len(resp.Contents)-1].Key + objResp.NextMarker = q.Marker = *resp.Contents[len(resp.Contents)-1].Key } return objResp, nil From 72506c7da32fd63e9bd2b79b3ab653ed4d2fa988 Mon Sep 17 00:00:00 2001 From: Tiago Peczenyj Date: Fri, 24 Aug 2018 11:46:02 +0200 Subject: [PATCH 2/3] fix store.go fix typo --- awss3/store.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awss3/store.go b/awss3/store.go index a89fb86..f5bbfb9 100644 --- a/awss3/store.go +++ b/awss3/store.go @@ -307,7 +307,9 @@ func (f *FS) List(ctx context.Context, q cloudstorage.Query) (*cloudstorage.Obje } if resp.IsTruncated != nil && *resp.IsTruncated { - objResp.NextMarker = q.Marker = *resp.Contents[len(resp.Contents)-1].Key + lastObj := *resp.Contents[len(resp.Contents)-1].Key + objResp.NextMarker = lastObj + q.Marker = lastObj } return objResp, nil From 7bc7cb22b6fef06f6254496bd7612b0bf623ce3e Mon Sep 17 00:00:00 2001 From: Aaron Raddon Date: Sat, 25 Aug 2018 13:48:59 -0700 Subject: [PATCH 3/3] Ensure we have failing tests refs #66 --- testutils/testutils.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/testutils/testutils.go b/testutils/testutils.go index 7cd9fb8..2ed7bc4 100644 --- a/testutils/testutils.go +++ b/testutils/testutils.go @@ -474,6 +474,7 @@ func ListObjsAndFolders(t TestingT, store cloudstorage.Store) { createObjects(names) + // Test that we can get the full list of 15 objects q := cloudstorage.NewQuery("list-test/") q.PageSize = 500 q.Sorted() @@ -483,6 +484,7 @@ func ListObjsAndFolders(t TestingT, store cloudstorage.Store) { assert.Equal(t, 15, len(objs), "incorrect list len. wanted 15 got %d", len(objs)) iter.Close() + // Now test the response from iter iter, _ = store.Objects(context.Background(), q) objr, err := cloudstorage.ObjectResponseFromIter(iter) assert.Equal(t, nil, err) @@ -495,6 +497,21 @@ func ListObjsAndFolders(t TestingT, store cloudstorage.Store) { assert.Equal(t, nil, err) assert.Equal(t, 15, len(objResp.Objects), "incorrect list len. wanted 15 got %d", len(objResp.Objects)) + // Now we are going to test page-sizing, ie set page size below known list + q = cloudstorage.NewQuery("list-test/") + q.PageSize = 3 + iter, _ = store.Objects(context.Background(), q) + objr, err = cloudstorage.ObjectResponseFromIter(iter) + // Even though page size=3, we actually want 15 because it is going + // to page through multiple pages. + assert.Equal(t, nil, err) + assert.Equal(t, 15, len(objr.Objects), "incorrect list len. wanted 15 got %d", len(objr.Objects)) + // Now ensure we can get only 3 + objr, err = store.List(context.Background(), q) + assert.Equal(t, nil, err) + assert.Equal(t, 3, len(objr.Objects), "incorrect list len. wanted 3 got %d", len(objr.Objects)) + assert.NotEqual(t, "", objr.NextMarker) + // Now we are going to re-run this test using an Object Iterator // that uses store.List() instead of store.Objects() q = cloudstorage.NewQuery("list-test/")