Skip to content

Use iter.Seq to iterate over ScanIterator #3348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ In `go-redis` we are aiming to support the last three releases of Redis. Current
- [Redis 7.4](https://raw.githubusercontent.com/redis/redis/7.4/00-RELEASENOTES) - using Redis Stack 7.4 for modules support
- [Redis 8.0](https://raw.githubusercontent.com/redis/redis/8.0/00-RELEASENOTES) - using Redis CE 8.0 where modules are included

Although the `go.mod` states it requires at minimum `go 1.18`, our CI is configured to run the tests against all three
Although the `go.mod` states it requires at minimum `go 1.23`, our CI is configured to run the tests against all three
versions of Redis and latest two versions of Go ([1.23](https://go.dev/doc/devel/release#go1.23.0),
[1.24](https://go.dev/doc/devel/release#go1.24.0)). We observe that some modules related test may not pass with
Redis Stack 7.2 and some commands are changed with Redis CE 8.0.
Expand Down
2 changes: 1 addition & 1 deletion example/del-keys-without-ttl/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/example/del-keys-without-ttl

go 1.18
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
4 changes: 2 additions & 2 deletions example/del-keys-without-ttl/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ func main() {
checker.Start(ctx)

iter := rdb.Scan(ctx, 0, "", 0).Iterator()
for iter.Next(ctx) {
checker.Add(iter.Val())
for val := range iter.Vals(ctx) {
checker.Add(val)
}
if err := iter.Err(); err != nil {
panic(err)
Expand Down
2 changes: 1 addition & 1 deletion example/hll/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/example/hll

go 1.18
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion example/hset-struct/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/example/scan-struct

go 1.18
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion example/lua-scripting/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/example/lua-scripting

go 1.18
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion example/redis-bloom/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/example/redis-bloom

go 1.18
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion example/scan-struct/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/example/scan-struct

go 1.18
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
8 changes: 4 additions & 4 deletions example_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -632,8 +632,8 @@ func Example_customCommand2() {

func ExampleScanIterator() {
iter := rdb.Scan(ctx, 0, "", 0).Iterator()
for iter.Next(ctx) {
fmt.Println(iter.Val())
for val := range iter.Vals(ctx) {
fmt.Println(val)
}
if err := iter.Err(); err != nil {
panic(err)
Expand All @@ -642,8 +642,8 @@ func ExampleScanIterator() {

func ExampleScanCmd_Iterator() {
iter := rdb.Scan(ctx, 0, "", 0).Iterator()
for iter.Next(ctx) {
fmt.Println(iter.Val())
for val := range iter.Vals(ctx) {
fmt.Println(val)
}
if err := iter.Err(); err != nil {
panic(err)
Expand Down
2 changes: 1 addition & 1 deletion extra/rediscensus/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/extra/rediscensus/v9

go 1.19
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion extra/rediscmd/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/extra/rediscmd/v9

go 1.19
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion extra/redisotel/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/extra/redisotel/v9

go 1.19
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion extra/redisprometheus/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/extra/redisprometheus/v9

go 1.19
go 1.23

replace github.com/redis/go-redis/v9 => ../..

Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/v9

go 1.18
go 1.23

require (
github.com/bsm/ginkgo/v2 v2.12.0
Expand Down
2 changes: 1 addition & 1 deletion internal/customvet/go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/redis/go-redis/internal/customvet

go 1.17
go 1.23

require golang.org/x/tools v0.5.0

Expand Down
43 changes: 43 additions & 0 deletions iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package redis

import (
"context"
"iter"
)

// ScanIterator is used to incrementally iterate over a collection of elements.
Expand All @@ -16,6 +17,8 @@ func (it *ScanIterator) Err() error {
}

// Next advances the cursor and returns true if more values can be read.
//
// Deprecated: support for native iterators has been added in go 1.23. Use Vals instead.
func (it *ScanIterator) Next(ctx context.Context) bool {
Comment on lines +20 to 22
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The Vals itself will be using Next. In general I would vote agains marking those as deprecated since they will continue to be used inside the library and if a developer would prefer to use them instead of the Vals I don't think we should discourage that by marking Next and Val as deprecated.
Let me know what you think.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think iterator via Vals is easier and more convenient than the combination Next, Val.

When you use Val, you need to know that Next must be called before it.
When you use Vals, you just iterate through the values.

Also the code of Vals could be much simpler without calls of Next, Val, without pos var handling, without extra cmd.Err check at each Val call.

// Vals returns iterator over key/field at the current cursor position.
func (it *ScanIterator) Vals(ctx context.Context) iter.Seq[string] {
	return func(yield func(string) bool) {
		if it.cmd.Err() != nil {
			return
		}

		for {
			for _, val := range it.cmd.page {
				if !yield(val) {
					return
				}
			}

			// Return if there is no more data to fetch.
			if it.cmd.cursor == 0 {
				return
			}

			// Fetch next page.
			var cursorIndex int
			switch it.cmd.args[0] {
			case "scan", "qscan":
				cursorIndex = 1
			default:
				cursorIndex = 2
			}

			it.cmd.args[cursorIndex] = it.cmd.cursor

			err := it.cmd.process(ctx, it.cmd)
			if err != nil {
				return
			}
		}
	}
}

they will continue to be used inside the library

In PR Next, Val is used only at this place, I left them in for backward compatibility only.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't disagree that an Iterator is more ergonomic. What I wanted to make sure is that we do not deprecate the current way of iterating over the values before version 10 of go-redis. If you feel strongly about deprecating, we can postpone this PR for when go-redis/v10 will be released.

// Instantly return on errors.
if it.cmd.Err() != nil {
Expand Down Expand Up @@ -57,10 +60,50 @@ func (it *ScanIterator) Next(ctx context.Context) bool {
}

// Val returns the key/field at the current cursor position.
//
// Deprecated: support for native iterators has been added in go 1.23. Use Vals instead.
func (it *ScanIterator) Val() string {
var v string
if it.cmd.Err() == nil && it.pos > 0 && it.pos <= len(it.cmd.page) {
v = it.cmd.page[it.pos-1]
}
return v
}

// Vals returns iterator over key/field at the current cursor position.
func (it *ScanIterator) Vals(ctx context.Context) iter.Seq[string] {
return func(yield func(string) bool) {
if it.cmd.Err() != nil {
return
}

for {
for _, val := range it.cmd.page {
if !yield(val) {
return
}
}

// Return if there is no more data to fetch.
if it.cmd.cursor == 0 {
return
}

// Fetch next page.
var cursorIndex int
switch it.cmd.args[0] {
case "scan", "qscan":
cursorIndex = 1
default:
cursorIndex = 2
}

it.cmd.args[cursorIndex] = it.cmd.cursor

err := it.cmd.process(ctx, it.cmd)
if err != nil {
return
}
}
}
}
97 changes: 89 additions & 8 deletions iterator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package redis_test

import (
"fmt"
"slices"

. "github.com/bsm/ginkgo/v2"
. "github.com/bsm/gomega"
Expand Down Expand Up @@ -52,13 +53,20 @@ var _ = Describe("ScanIterator", func() {
Expect(client.Close()).NotTo(HaveOccurred())
})

It("should scan across empty DBs", func() {
It("should scan across empty DBs using Next", func() {
iter := client.Scan(ctx, 0, "", 10).Iterator()
Expect(iter.Next(ctx)).To(BeFalse())
Expect(iter.Err()).NotTo(HaveOccurred())
})

It("should scan across one page", func() {
It("should scan across empty DBs using Vals", func() {
iter := client.Scan(ctx, 0, "", 10).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(vals).To(BeEmpty())
Expect(iter.Err()).NotTo(HaveOccurred())
})

It("should scan across one page using Next", func() {
Expect(seed(7)).NotTo(HaveOccurred())

var vals []string
Expand All @@ -70,7 +78,16 @@ var _ = Describe("ScanIterator", func() {
Expect(vals).To(ConsistOf([]string{"K01", "K02", "K03", "K04", "K05", "K06", "K07"}))
})

It("should scan across multiple pages", func() {
It("should scan across one page using Vals", func() {
Expect(seed(7)).NotTo(HaveOccurred())

iter := client.Scan(ctx, 0, "", 0).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(ConsistOf([]string{"K01", "K02", "K03", "K04", "K05", "K06", "K07"}))
})

It("should scan across multiple pages using Next", func() {
Expect(seed(71)).NotTo(HaveOccurred())

var vals []string
Expand All @@ -84,7 +101,18 @@ var _ = Describe("ScanIterator", func() {
Expect(vals).To(ContainElement("K71"))
})

It("should hscan across multiple pages", func() {
It("should scan across multiple pages using Vals", func() {
Expect(seed(71)).NotTo(HaveOccurred())

iter := client.Scan(ctx, 0, "", 10).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(71))
Expect(vals).To(ContainElement("K01"))
Expect(vals).To(ContainElement("K71"))
})

It("should hscan across multiple pages using Next", func() {
SkipBeforeRedisVersion(7.4, "doesn't work with older redis stack images")
Expect(hashSeed(71)).NotTo(HaveOccurred())

Expand All @@ -100,7 +128,20 @@ var _ = Describe("ScanIterator", func() {
Expect(vals).To(ContainElement("x"))
})

It("should hscan without values across multiple pages", Label("NonRedisEnterprise"), func() {
It("should hscan across multiple pages using Vals", func() {
SkipBeforeRedisVersion(7.4, "doesn't work with older redis stack images")
Expect(hashSeed(71)).NotTo(HaveOccurred())

iter := client.HScan(ctx, hashKey, 0, "", 10).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(71 * 2))
Expect(vals).To(ContainElement("K01"))
Expect(vals).To(ContainElement("K71"))
Expect(vals).To(ContainElement("x"))
})

It("should hscan without values across multiple pages using Next", Label("NonRedisEnterprise"), func() {
SkipBeforeRedisVersion(7.4, "doesn't work with older redis stack images")
Expect(hashSeed(71)).NotTo(HaveOccurred())

Expand All @@ -116,7 +157,20 @@ var _ = Describe("ScanIterator", func() {
Expect(vals).NotTo(ContainElement("x"))
})

It("should scan to page borders", func() {
It("should hscan without values across multiple pages using Vals", Label("NonRedisEnterprise"), func() {
SkipBeforeRedisVersion(7.4, "doesn't work with older redis stack images")
Expect(hashSeed(71)).NotTo(HaveOccurred())

iter := client.HScanNoValues(ctx, hashKey, 0, "", 10).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(71))
Expect(vals).To(ContainElement("K01"))
Expect(vals).To(ContainElement("K71"))
Expect(vals).NotTo(ContainElement("x"))
})

It("should scan to page borders using Next", func() {
Expect(seed(20)).NotTo(HaveOccurred())

var vals []string
Expand All @@ -128,7 +182,16 @@ var _ = Describe("ScanIterator", func() {
Expect(vals).To(HaveLen(20))
})

It("should scan with match", func() {
It("should scan to page borders using Vals", func() {
Expect(seed(20)).NotTo(HaveOccurred())

iter := client.Scan(ctx, 0, "", 10).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(20))
})

It("should scan with match using Next", func() {
Expect(seed(33)).NotTo(HaveOccurred())

var vals []string
Expand All @@ -140,7 +203,16 @@ var _ = Describe("ScanIterator", func() {
Expect(vals).To(HaveLen(13))
})

It("should scan with match across empty pages", func() {
It("should scan with match using Vals", func() {
Expect(seed(33)).NotTo(HaveOccurred())

iter := client.Scan(ctx, 0, "K*2*", 10).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(13))
})

It("should scan with match across empty pages using Next", func() {
Expect(extraSeed(2, 10)).NotTo(HaveOccurred())

var vals []string
Expand All @@ -151,4 +223,13 @@ var _ = Describe("ScanIterator", func() {
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(2))
})

It("should scan with match across empty pages using Vals", func() {
Expect(extraSeed(2, 10)).NotTo(HaveOccurred())

iter := client.Scan(ctx, 0, "K*", 1).Iterator()
vals := slices.Collect(iter.Vals(ctx))
Expect(iter.Err()).NotTo(HaveOccurred())
Expect(vals).To(HaveLen(2))
})
})
Loading