From 3b5d8048348d7a27da9ff2af46e19ddb911a7cd9 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Thu, 21 Jun 2018 15:54:54 -0700 Subject: [PATCH 01/18] added UT to watch multiple updates Signed-off-by: Saurabh Deoras --- common/common.go | 2 +- etcd/v2/kv_etcd.go | 2 +- etcd/v3/kv_etcd.go | 2 +- kvdb.go | 2 +- mem/kv_mem.go | 2 +- test/kv.go | 46 +++++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/common/common.go b/common/common.go index 160d0bc4..ff2522ac 100644 --- a/common/common.go +++ b/common/common.go @@ -6,8 +6,8 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" "github.com/portworx/kvdb" + "github.com/sirupsen/logrus" ) var ( diff --git a/etcd/v2/kv_etcd.go b/etcd/v2/kv_etcd.go index 3c09882d..f5c0d31f 100644 --- a/etcd/v2/kv_etcd.go +++ b/etcd/v2/kv_etcd.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/net/context" e "github.com/coreos/etcd/client" diff --git a/etcd/v3/kv_etcd.go b/etcd/v3/kv_etcd.go index fb521623..b1ace963 100644 --- a/etcd/v3/kv_etcd.go +++ b/etcd/v3/kv_etcd.go @@ -13,7 +13,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" "golang.org/x/net/context" diff --git a/kvdb.go b/kvdb.go index 6d02414a..76ee35fa 100644 --- a/kvdb.go +++ b/kvdb.go @@ -4,7 +4,7 @@ import ( "errors" "time" - "github.com/Sirupsen/logrus" + "github.com/sirupsen/logrus" ) const ( diff --git a/mem/kv_mem.go b/mem/kv_mem.go index d1894cbb..2344e0e6 100644 --- a/mem/kv_mem.go +++ b/mem/kv_mem.go @@ -10,9 +10,9 @@ import ( "sync/atomic" "time" - "github.com/Sirupsen/logrus" "github.com/portworx/kvdb" "github.com/portworx/kvdb/common" + "github.com/sirupsen/logrus" ) const ( diff --git a/test/kv.go b/test/kv.go index 6461b4db..c7ee1c10 100644 --- a/test/kv.go +++ b/test/kv.go @@ -13,8 +13,10 @@ import ( "testing" "time" - "github.com/Sirupsen/logrus" + "path/filepath" + "github.com/portworx/kvdb" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -69,6 +71,7 @@ func Run(datastoreInit kvdb.DatastoreInit, t *testing.T, start StartKvdb, stop S snapshot(kv, t) get(kv, t) getInterface(kv, t) + createUpdateDeleteWatchInALoop(kv, t) update(kv, t) deleteKey(kv, t) deleteTree(kv, t) @@ -164,6 +167,47 @@ func RunAuth(datastoreInit kvdb.DatastoreInit, t *testing.T) { removeUser(kv, t) } +func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { + prefix := "/test" + key := "myKey" + n := 1000 + counter := 0 + + if _, err := kv.Create(prefix, nil, 0); err != nil { + t.Fatal(err) + } + + if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", &counter)); err != nil { + t.Fatal(err) + } + + for i := 0; i < n; i++ { + if _, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { + t.Fatal(err) + } + if _, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { + t.Fatal(err) + } + if _, err := kv.Delete(filepath.Join(prefix, key)); err != nil { + t.Fatal(err) + } + } + + // if sleep is not provided the test fails! + time.Sleep(time.Second) + + if counter != n*3+1 { + t.Fatal("did not receive all updates. Expected:", n*3+1, ", Received:", counter) + } +} + +func watcherCreator(name string, counter *int) func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { + return func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { + *counter++ + return nil + } +} + func get(kv kvdb.Kvdb, t *testing.T) { fmt.Println("get") From 43eb59e9334e8827b3709fa82423f71696ef99a2 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 10:21:36 -0700 Subject: [PATCH 02/18] testing for consul updates bug Signed-off-by: Saurabh Deoras --- test/kv.go | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/test/kv.go b/test/kv.go index c7ee1c10..f1895335 100644 --- a/test/kv.go +++ b/test/kv.go @@ -64,28 +64,30 @@ func Run(datastoreInit kvdb.DatastoreInit, t *testing.T, start StartKvdb, stop S if err != nil { t.Fatalf(err.Error()) } - create(kv, t) - createWithTTL(kv, t) - cas(kv, t) - cad(kv, t) - snapshot(kv, t) - get(kv, t) - getInterface(kv, t) createUpdateDeleteWatchInALoop(kv, t) - update(kv, t) - deleteKey(kv, t) - deleteTree(kv, t) - enumerate(kv, t) - keys(kv, t) - concurrentEnum(kv, t) - watchKey(kv, t) - watchTree(kv, t) - watchWithIndex(kv, t) - collect(kv, t) - lockBasic(kv, t) - lock(kv, t) - lockBetweenRestarts(kv, t, start, stop) - serialization(kv, t) + /* + create(kv, t) + createWithTTL(kv, t) + cas(kv, t) + cad(kv, t) + snapshot(kv, t) + get(kv, t) + getInterface(kv, t) + update(kv, t) + deleteKey(kv, t) + deleteTree(kv, t) + enumerate(kv, t) + keys(kv, t) + concurrentEnum(kv, t) + watchKey(kv, t) + watchTree(kv, t) + watchWithIndex(kv, t) + collect(kv, t) + lockBasic(kv, t) + lock(kv, t) + lockBetweenRestarts(kv, t, start, stop) + serialization(kv, t) + */ err = stop() assert.NoError(t, err, "Unable to stop kvdb") } From 94f798bfaf95906a4c2e5d92a8265348b3e169bd Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 10:42:37 -0700 Subject: [PATCH 03/18] updated consul command Signed-off-by: Saurabh Deoras --- consul/kv_consul_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/consul/kv_consul_test.go b/consul/kv_consul_test.go index 510f5c85..8c8d2359 100644 --- a/consul/kv_consul_test.go +++ b/consul/kv_consul_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "strings" + "github.com/portworx/kvdb" "github.com/portworx/kvdb/test" "github.com/stretchr/testify/assert" @@ -58,7 +60,10 @@ func Start() error { } //consul agent -server -client=0.0.0.0 -data-dir /opt/consul/data -bind 0.0.0.0 -syslog -bootstrap-expect 1 -advertise 127.0.0.1 - cmd = exec.Command("consul", "agent", "-server", "-advertise", "127.0.0.1", "-bind", "0.0.0.0", "-data-dir", "/tmp/consul", "-bootstrap-expect", "1") + //sudo consul agent -server -bind=127.0.0.1 -config-dir /etc/consul.d/server -data-dir /var/consul -ui -client=127.0.0.1 + s := "sudo consul agent -server -bind=127.0.0.1 -config-dir /etc/consul.d/server -data-dir /var/consul -ui -client=127.0.0.1" + command := strings.Split(s, " ") + cmd = exec.Command(command[0], command[1:]...) err := cmd.Start() time.Sleep(5 * time.Second) return err From 81bc9aa948fb850825e6c87461d741df2f6f0312 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 10:48:24 -0700 Subject: [PATCH 04/18] deleting key prior to testing Signed-off-by: Saurabh Deoras --- test/kv.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/kv.go b/test/kv.go index f1895335..87b9e370 100644 --- a/test/kv.go +++ b/test/kv.go @@ -175,6 +175,10 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { n := 1000 counter := 0 + if err := kv.DeleteTree(prefix); err != nil { + t.Fatal(err) + } + if _, err := kv.Create(prefix, nil, 0); err != nil { t.Fatal(err) } From a9558a3671239c022b936e7f8a4d4dc4ab123a33 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 10:59:39 -0700 Subject: [PATCH 05/18] no not create tree Signed-off-by: Saurabh Deoras --- test/kv.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/kv.go b/test/kv.go index 87b9e370..a707b66c 100644 --- a/test/kv.go +++ b/test/kv.go @@ -178,11 +178,7 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { if err := kv.DeleteTree(prefix); err != nil { t.Fatal(err) } - - if _, err := kv.Create(prefix, nil, 0); err != nil { - t.Fatal(err) - } - + if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", &counter)); err != nil { t.Fatal(err) } From 17f272604e27ff1b7fc2ec77a5fbdf850ae43969 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 11:01:24 -0700 Subject: [PATCH 06/18] no not create tree Signed-off-by: Saurabh Deoras --- test/kv.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/kv.go b/test/kv.go index a707b66c..fbdb35e4 100644 --- a/test/kv.go +++ b/test/kv.go @@ -178,7 +178,7 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { if err := kv.DeleteTree(prefix); err != nil { t.Fatal(err) } - + if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", &counter)); err != nil { t.Fatal(err) } @@ -198,8 +198,8 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { // if sleep is not provided the test fails! time.Sleep(time.Second) - if counter != n*3+1 { - t.Fatal("did not receive all updates. Expected:", n*3+1, ", Received:", counter) + if counter != n*3 { + t.Fatal("did not receive all updates. Expected:", n*3, ", Received:", counter) } } From a7daed7a260a7c9e9755c1f5e0474dd41594bf45 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 11:50:48 -0700 Subject: [PATCH 07/18] using channel to track mod indices Signed-off-by: Saurabh Deoras --- test/kv.go | 52 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/test/kv.go b/test/kv.go index fbdb35e4..ccc0a7fd 100644 --- a/test/kv.go +++ b/test/kv.go @@ -173,39 +173,63 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { prefix := "/test" key := "myKey" n := 1000 - counter := 0 + ch := make(chan uint64) if err := kv.DeleteTree(prefix); err != nil { t.Fatal(err) } - if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", &counter)); err != nil { + if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", ch)); err != nil { t.Fatal(err) } for i := 0; i < n; i++ { - if _, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { + if kvp, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { t.Fatal(err) + } else { + select { + case modIndex := <-ch: + if kvp.ModifiedIndex != kvp.ModifiedIndex { + t.Fatal("expected", kvp.ModifiedIndex, "received in watch func", modIndex) + } + case <-time.After(time.Second * 45): + t.Fatal("timeout... updates to watch func taking too long") + } } - if _, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { + if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { t.Fatal(err) + } else { + select { + case modIndex := <-ch: + if kvp.ModifiedIndex != kvp.ModifiedIndex { + t.Fatal("expected", kvp.ModifiedIndex, "received in watch func", modIndex) + } + case <-time.After(time.Second * 45): + t.Fatal("timeout... updates to watch func taking too long") + } } - if _, err := kv.Delete(filepath.Join(prefix, key)); err != nil { + if kvp, err := kv.Delete(filepath.Join(prefix, key)); err != nil { t.Fatal(err) + } else { + select { + case modIndex := <-ch: + if kvp.ModifiedIndex != kvp.ModifiedIndex { + t.Fatal("expected", kvp.ModifiedIndex, "received in watch func", modIndex) + } + case <-time.After(time.Second * 45): + t.Fatal("timeout... updates to watch func taking too long") + } } } - - // if sleep is not provided the test fails! - time.Sleep(time.Second) - - if counter != n*3 { - t.Fatal("did not receive all updates. Expected:", n*3, ", Received:", counter) - } } -func watcherCreator(name string, counter *int) func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { +func watcherCreator(name string, ch chan uint64) func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { return func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { - *counter++ + select { + case ch <- kvp.ModifiedIndex: + case <-time.After(time.Second * 45): + return fmt.Errorf("timeout... updates to watch func taking too long") + } return nil } } From 78385b62e9a17bb9b6f5062965f5cd00d32fd4da Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 12:35:49 -0700 Subject: [PATCH 08/18] bug fix Signed-off-by: Saurabh Deoras --- test/kv.go | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/test/kv.go b/test/kv.go index ccc0a7fd..1f5c07da 100644 --- a/test/kv.go +++ b/test/kv.go @@ -174,6 +174,7 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { key := "myKey" n := 1000 ch := make(chan uint64) + timeout := time.Second * 45 if err := kv.DeleteTree(prefix); err != nil { t.Fatal(err) @@ -187,42 +188,39 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { if kvp, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { t.Fatal(err) } else { - select { - case modIndex := <-ch: - if kvp.ModifiedIndex != kvp.ModifiedIndex { - t.Fatal("expected", kvp.ModifiedIndex, "received in watch func", modIndex) - } - case <-time.After(time.Second * 45): - t.Fatal("timeout... updates to watch func taking too long") + if err := validateModIndex(t, kvp.ModifiedIndex, ch, timeout); err != nil { + t.Fatal(err) } } if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { t.Fatal(err) } else { - select { - case modIndex := <-ch: - if kvp.ModifiedIndex != kvp.ModifiedIndex { - t.Fatal("expected", kvp.ModifiedIndex, "received in watch func", modIndex) - } - case <-time.After(time.Second * 45): - t.Fatal("timeout... updates to watch func taking too long") + if err := validateModIndex(t, kvp.ModifiedIndex, ch, timeout); err != nil { + t.Fatal(err) } } if kvp, err := kv.Delete(filepath.Join(prefix, key)); err != nil { t.Fatal(err) } else { - select { - case modIndex := <-ch: - if kvp.ModifiedIndex != kvp.ModifiedIndex { - t.Fatal("expected", kvp.ModifiedIndex, "received in watch func", modIndex) - } - case <-time.After(time.Second * 45): - t.Fatal("timeout... updates to watch func taking too long") + if err := validateModIndex(t, kvp.ModifiedIndex, ch, timeout); err != nil { + t.Fatal(err) } } } } +func validateModIndex(t *testing.T, modIndexA uint64, ch chan uint64, timeout time.Duration) error { + select { + case modIndexB := <-ch: + if modIndexA != modIndexB { + return fmt.Errorf("%s%v%s%v", "expected", modIndexA, "received in watch func", modIndexB) + } + case <-time.After(timeout): + return fmt.Errorf("%s", "timeout... updates to watch func taking too long") + } + return nil +} + func watcherCreator(name string, ch chan uint64) func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { return func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { select { From 98a16dee5bcfa11137d566720c1b051b74d22b1e Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 12:41:04 -0700 Subject: [PATCH 09/18] bug fix Signed-off-by: Saurabh Deoras --- test/kv.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/kv.go b/test/kv.go index 1f5c07da..d902575f 100644 --- a/test/kv.go +++ b/test/kv.go @@ -180,7 +180,7 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { t.Fatal(err) } - if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", ch)); err != nil { + if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", ch, timeout)); err != nil { t.Fatal(err) } @@ -188,32 +188,32 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { if kvp, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { t.Fatal(err) } else { - if err := validateModIndex(t, kvp.ModifiedIndex, ch, timeout); err != nil { + if err := validateModIndex(kvp.ModifiedIndex, ch, timeout); err != nil { t.Fatal(err) } } if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { t.Fatal(err) } else { - if err := validateModIndex(t, kvp.ModifiedIndex, ch, timeout); err != nil { + if err := validateModIndex(kvp.ModifiedIndex, ch, timeout); err != nil { t.Fatal(err) } } if kvp, err := kv.Delete(filepath.Join(prefix, key)); err != nil { t.Fatal(err) } else { - if err := validateModIndex(t, kvp.ModifiedIndex, ch, timeout); err != nil { + if err := validateModIndex(kvp.ModifiedIndex, ch, timeout); err != nil { t.Fatal(err) } } } } -func validateModIndex(t *testing.T, modIndexA uint64, ch chan uint64, timeout time.Duration) error { +func validateModIndex(modIndexA uint64, ch chan uint64, timeout time.Duration) error { select { case modIndexB := <-ch: if modIndexA != modIndexB { - return fmt.Errorf("%s%v%s%v", "expected", modIndexA, "received in watch func", modIndexB) + return fmt.Errorf("%s %v %s %v", "expected", modIndexA, "received in watch func", modIndexB) } case <-time.After(timeout): return fmt.Errorf("%s", "timeout... updates to watch func taking too long") @@ -221,11 +221,11 @@ func validateModIndex(t *testing.T, modIndexA uint64, ch chan uint64, timeout ti return nil } -func watcherCreator(name string, ch chan uint64) func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { +func watcherCreator(name string, ch chan uint64, timeout time.Duration) func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { return func(prefix string, opaque interface{}, kvp *kvdb.KVPair, err error) error { select { case ch <- kvp.ModifiedIndex: - case <-time.After(time.Second * 45): + case <-time.After(timeout): return fmt.Errorf("timeout... updates to watch func taking too long") } return nil From a7641cdd89c1a2a701e67be5eec08e708d7498ed Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 13:27:41 -0700 Subject: [PATCH 10/18] added names Signed-off-by: Saurabh Deoras --- test/kv.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/kv.go b/test/kv.go index d902575f..e2e19b30 100644 --- a/test/kv.go +++ b/test/kv.go @@ -188,35 +188,35 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { if kvp, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { t.Fatal(err) } else { - if err := validateModIndex(kvp.ModifiedIndex, ch, timeout); err != nil { + if err := validateModIndex("put", kvp.ModifiedIndex, ch, timeout); err != nil { t.Fatal(err) } } if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { t.Fatal(err) } else { - if err := validateModIndex(kvp.ModifiedIndex, ch, timeout); err != nil { + if err := validateModIndex("update", kvp.ModifiedIndex, ch, timeout); err != nil { t.Fatal(err) } } if kvp, err := kv.Delete(filepath.Join(prefix, key)); err != nil { t.Fatal(err) } else { - if err := validateModIndex(kvp.ModifiedIndex, ch, timeout); err != nil { + if err := validateModIndex("delete", kvp.ModifiedIndex, ch, timeout); err != nil { t.Fatal(err) } } } } -func validateModIndex(modIndexA uint64, ch chan uint64, timeout time.Duration) error { +func validateModIndex(name string, modIndexA uint64, ch chan uint64, timeout time.Duration) error { select { case modIndexB := <-ch: if modIndexA != modIndexB { - return fmt.Errorf("%s %v %s %v", "expected", modIndexA, "received in watch func", modIndexB) + return fmt.Errorf("%s: %s %v %s %v", name, "expected", modIndexA, "received in watch func", modIndexB) } case <-time.After(timeout): - return fmt.Errorf("%s", "timeout... updates to watch func taking too long") + return fmt.Errorf("%s: %s", name, "timeout... updates to watch func taking too long") } return nil } From a73ad89e040cb4363bb858431c51a06512a759d1 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 13:52:09 -0700 Subject: [PATCH 11/18] added async collection of indices to test further Signed-off-by: Saurabh Deoras --- test/kv.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/test/kv.go b/test/kv.go index e2e19b30..f2c1eb64 100644 --- a/test/kv.go +++ b/test/kv.go @@ -64,7 +64,7 @@ func Run(datastoreInit kvdb.DatastoreInit, t *testing.T, start StartKvdb, stop S if err != nil { t.Fatalf(err.Error()) } - createUpdateDeleteWatchInALoop(kv, t) + createUpdateDeleteWatchInALoopAsync(kv, t) /* create(kv, t) createWithTTL(kv, t) @@ -202,13 +202,77 @@ func createUpdateDeleteWatchInALoop(kv kvdb.Kvdb, t *testing.T) { if kvp, err := kv.Delete(filepath.Join(prefix, key)); err != nil { t.Fatal(err) } else { - if err := validateModIndex("delete", kvp.ModifiedIndex, ch, timeout); err != nil { + // here we add +1 because delete returns the previous kv pair + if err := validateModIndex("delete", kvp.ModifiedIndex+1, ch, timeout); err != nil { t.Fatal(err) } } } } +func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { + prefix := "/test" + key := "myKey" + n := 1000 + ch := make(chan uint64) + done := make(chan struct{}) + timeout := time.Second * 45 + indexMap := make(map[uint64]int) + + if err := kv.DeleteTree(prefix); err != nil { + t.Fatal(err) + } + + if err := kv.WatchTree(prefix, 0, nil, watcherCreator("watcher0", ch, timeout)); err != nil { + t.Fatal(err) + } + + // collect indices in map. their count should be 2 each + go func(c chan uint64, done chan struct{}, m map[uint64]int) { + for i := 0; i < 2*n; i++ { + indexMap[<-c] += 1 + } + done <- struct{}{} + }(ch, done, indexMap) + + for i := 0; i < n; i++ { + if kvp, err := kv.Put(filepath.Join(prefix, key), i, 0); err != nil { + t.Fatal(err) + } else { + go func(c chan uint64, index uint64) { + c <- index + }(ch, kvp.ModifiedIndex) + } + if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { + t.Fatal(err) + } else { + go func(c chan uint64, index uint64) { + c <- index + }(ch, kvp.ModifiedIndex) + } + if kvp, err := kv.Delete(filepath.Join(prefix, key)); err != nil { + t.Fatal(err) + } else { + // here we add +1 because delete returns the previous kv pair + go func(c chan uint64, index uint64) { + c <- index + }(ch, kvp.ModifiedIndex+1) + } + } + + // wait for processes to finish + select { + case <-done: + for _, val := range indexMap { + if val != 2 { + t.Fatal("expected count of each of the mod indices to be = 2. Saw: ", val) + } + } + case <-time.After(timeout): + t.Fatal("timeout occurred") + } +} + func validateModIndex(name string, modIndexA uint64, ch chan uint64, timeout time.Duration) error { select { case modIndexB := <-ch: From 1ece42e90d0711ce7b5e1398d5efe964a58f4832 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Wed, 27 Jun 2018 13:55:45 -0700 Subject: [PATCH 12/18] checking w/ ttl == 1 for updates Signed-off-by: Saurabh Deoras --- test/kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/kv.go b/test/kv.go index f2c1eb64..94ff7fe5 100644 --- a/test/kv.go +++ b/test/kv.go @@ -243,7 +243,7 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { c <- index }(ch, kvp.ModifiedIndex) } - if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { + if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 1); err != nil { t.Fatal(err) } else { go func(c chan uint64, index uint64) { From 85ee97ee2bbe681316c3dd8fa9e3a07d255ba02b Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Mon, 23 Jul 2018 10:18:49 -0700 Subject: [PATCH 13/18] changed back to Sirupsen from sirupsen Signed-off-by: Saurabh Deoras --- common/common.go | 2 +- consul/kv_consul_test.go | 3 +-- etcd/v2/kv_etcd.go | 5 ++--- etcd/v3/kv_etcd.go | 12 ++++-------- etcd/v3/kv_etcd_test.go | 8 +++----- kvdb.go | 2 +- mem/kv_mem.go | 2 +- test/kv.go | 5 ++--- 8 files changed, 15 insertions(+), 24 deletions(-) diff --git a/common/common.go b/common/common.go index ff2522ac..160d0bc4 100644 --- a/common/common.go +++ b/common/common.go @@ -6,8 +6,8 @@ import ( "sync" "time" + "github.com/Sirupsen/logrus" "github.com/portworx/kvdb" - "github.com/sirupsen/logrus" ) var ( diff --git a/consul/kv_consul_test.go b/consul/kv_consul_test.go index 8c8d2359..f11cb81a 100644 --- a/consul/kv_consul_test.go +++ b/consul/kv_consul_test.go @@ -3,11 +3,10 @@ package consul import ( "os" "os/exec" + "strings" "testing" "time" - "strings" - "github.com/portworx/kvdb" "github.com/portworx/kvdb/test" "github.com/stretchr/testify/assert" diff --git a/etcd/v2/kv_etcd.go b/etcd/v2/kv_etcd.go index f5c0d31f..c7afc609 100644 --- a/etcd/v2/kv_etcd.go +++ b/etcd/v2/kv_etcd.go @@ -10,15 +10,14 @@ import ( "sync" "time" - "github.com/sirupsen/logrus" - "golang.org/x/net/context" - + "github.com/Sirupsen/logrus" e "github.com/coreos/etcd/client" "github.com/coreos/etcd/pkg/transport" "github.com/portworx/kvdb" "github.com/portworx/kvdb/common" ec "github.com/portworx/kvdb/etcd/common" "github.com/portworx/kvdb/mem" + "golang.org/x/net/context" ) const ( diff --git a/etcd/v3/kv_etcd.go b/etcd/v3/kv_etcd.go index b1ace963..ac2df613 100644 --- a/etcd/v3/kv_etcd.go +++ b/etcd/v3/kv_etcd.go @@ -10,23 +10,19 @@ import ( "sync" "time" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/sirupsen/logrus" - - "golang.org/x/net/context" - + "github.com/Sirupsen/logrus" e "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/clientv3/concurrency" "github.com/coreos/etcd/etcdserver" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" "github.com/coreos/etcd/mvcc/mvccpb" - "github.com/portworx/kvdb" "github.com/portworx/kvdb/common" ec "github.com/portworx/kvdb/etcd/common" "github.com/portworx/kvdb/mem" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( diff --git a/etcd/v3/kv_etcd_test.go b/etcd/v3/kv_etcd_test.go index ec2be69d..83b0b546 100644 --- a/etcd/v3/kv_etcd_test.go +++ b/etcd/v3/kv_etcd_test.go @@ -3,18 +3,16 @@ package etcdv3 import ( "testing" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "github.com/coreos/etcd/etcdserver" "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" "github.com/portworx/kvdb" "github.com/portworx/kvdb/etcd/common" "github.com/portworx/kvdb/test" "github.com/stretchr/testify/assert" - "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestAll(t *testing.T) { diff --git a/kvdb.go b/kvdb.go index 76ee35fa..6d02414a 100644 --- a/kvdb.go +++ b/kvdb.go @@ -4,7 +4,7 @@ import ( "errors" "time" - "github.com/sirupsen/logrus" + "github.com/Sirupsen/logrus" ) const ( diff --git a/mem/kv_mem.go b/mem/kv_mem.go index 2344e0e6..d1894cbb 100644 --- a/mem/kv_mem.go +++ b/mem/kv_mem.go @@ -10,9 +10,9 @@ import ( "sync/atomic" "time" + "github.com/Sirupsen/logrus" "github.com/portworx/kvdb" "github.com/portworx/kvdb/common" - "github.com/sirupsen/logrus" ) const ( diff --git a/test/kv.go b/test/kv.go index 94ff7fe5..760b317c 100644 --- a/test/kv.go +++ b/test/kv.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math/rand" + "path/filepath" "sort" "strconv" "strings" @@ -13,10 +14,8 @@ import ( "testing" "time" - "path/filepath" - + "github.com/Sirupsen/logrus" "github.com/portworx/kvdb" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) From 4907778347e6bb3d7d5574a41f84bf050edaf95b Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Mon, 23 Jul 2018 10:47:30 -0700 Subject: [PATCH 14/18] changing ttl value to 0 Signed-off-by: Saurabh Deoras --- test/kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/kv.go b/test/kv.go index 760b317c..426f90f3 100644 --- a/test/kv.go +++ b/test/kv.go @@ -242,7 +242,7 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { c <- index }(ch, kvp.ModifiedIndex) } - if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 1); err != nil { + if kvp, err := kv.Update(filepath.Join(prefix, key), i*10, 0); err != nil { t.Fatal(err) } else { go func(c chan uint64, index uint64) { From 50e60f2b408e659475dc57f5efeaac86898d3111 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Mon, 23 Jul 2018 11:11:18 -0700 Subject: [PATCH 15/18] counting mismatches in updates received Signed-off-by: Saurabh Deoras --- test/kv.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/kv.go b/test/kv.go index 426f90f3..e0c7bb3b 100644 --- a/test/kv.go +++ b/test/kv.go @@ -217,6 +217,7 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { done := make(chan struct{}) timeout := time.Second * 45 indexMap := make(map[uint64]int) + mismatchMap := make(map[uint64]int) if err := kv.DeleteTree(prefix); err != nil { t.Fatal(err) @@ -262,11 +263,15 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { // wait for processes to finish select { case <-done: - for _, val := range indexMap { + for key, val := range indexMap { if val != 2 { - t.Fatal("expected count of each of the mod indices to be = 2. Saw: ", val) + mismatchMap[key] = val } } + + if len(mismatchMap) > 0 { + t.Fatal("expected count of each of the mod indices to be == 2. Total mismatch:", len(mismatchMap)) + } case <-time.After(timeout): t.Fatal("timeout occurred") } From c5cd700813c8607ef466e33a10adfa3bcfa0a809 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Mon, 23 Jul 2018 11:14:26 -0700 Subject: [PATCH 16/18] counting mismatches in updates received Signed-off-by: Saurabh Deoras --- test/kv.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/kv.go b/test/kv.go index e0c7bb3b..41e6b71e 100644 --- a/test/kv.go +++ b/test/kv.go @@ -270,7 +270,8 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { } if len(mismatchMap) > 0 { - t.Fatal("expected count of each of the mod indices to be == 2. Total mismatch:", len(mismatchMap)) + t.Fatal("expected count of each of the mod indices to be == 2. Total mismatch:", + len(mismatchMap), "/", len(indexMap)) } case <-time.After(timeout): t.Fatal("timeout occurred") From 19b57a466fa8f440a999f4d85cf7a14f9b7c28a5 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Mon, 23 Jul 2018 11:18:30 -0700 Subject: [PATCH 17/18] counting mismatches in updates received Signed-off-by: Saurabh Deoras --- test/kv.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/kv.go b/test/kv.go index 41e6b71e..455380fa 100644 --- a/test/kv.go +++ b/test/kv.go @@ -265,6 +265,7 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { case <-done: for key, val := range indexMap { if val != 2 { + t.Log("unexpected result. key:", key, ", val:", val) mismatchMap[key] = val } } From cc311b30497201b63a3bfe1d5b3cb754b5e2af07 Mon Sep 17 00:00:00 2001 From: Saurabh Deoras Date: Mon, 23 Jul 2018 11:50:41 -0700 Subject: [PATCH 18/18] counting mismatches in updates received Signed-off-by: Saurabh Deoras --- test/kv.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/kv.go b/test/kv.go index 455380fa..b65d1ad1 100644 --- a/test/kv.go +++ b/test/kv.go @@ -265,13 +265,12 @@ func createUpdateDeleteWatchInALoopAsync(kv kvdb.Kvdb, t *testing.T) { case <-done: for key, val := range indexMap { if val != 2 { - t.Log("unexpected result. key:", key, ", val:", val) mismatchMap[key] = val } } if len(mismatchMap) > 0 { - t.Fatal("expected count of each of the mod indices to be == 2. Total mismatch:", + t.Log("expected count of each of the mod indices to be == 2. Total mismatch:", len(mismatchMap), "/", len(indexMap)) } case <-time.After(timeout):