Skip to content

Commit

Permalink
Rewrite Bound Selector (#357)
Browse files Browse the repository at this point in the history 
Also fix range iterator order
  • Loading branch information
@iskakaushik
iskakaushik authored Aug 27, 2023
1 parent 6682ea1 commit 2190dec
Show file tree
Hide file tree
Showing 6 changed files with 437 additions and 207 deletions.
64 changes: 36 additions & 28 deletions flow/concurrency/bound_selector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,46 +7,54 @@ import (
)

type BoundSelector struct {
ctx workflow.Context
limit int
selector workflow.Selector
futures map[workflow.Future]struct{}
ferrors []error
ctx workflow.Context
limit workflow.Channel
statusCh workflow.Channel
numFutures int
}

func NewBoundSelector(limit int, ctx workflow.Context) *BoundSelector {
func NewBoundSelector(limit int, total int, ctx workflow.Context) *BoundSelector {
return &BoundSelector{
ctx: ctx,
limit: limit,
selector: workflow.NewSelector(ctx),
futures: make(map[workflow.Future]struct{}),
ferrors: make([]error, 0),
ctx: ctx,
limit: workflow.NewBufferedChannel(ctx, limit),
statusCh: workflow.NewBufferedChannel(ctx, total),
numFutures: 0,
}
}

func (s *BoundSelector) AddFuture(future workflow.Future, f func(workflow.Future) error) {
if len(s.futures) >= s.limit {
s.selector.Select(s.ctx)
}

s.futures[future] = struct{}{}
s.selector.AddFuture(future, func(ready workflow.Future) {
delete(s.futures, ready)

err := f(ready)
if err != nil {
s.ferrors = append(s.ferrors, err)
}
func (s *BoundSelector) SpawnChild(chCtx workflow.Context, w interface{}, args ...interface{}) {
s.numFutures++
workflow.Go(s.ctx, func(ctx workflow.Context) {
s.limit.Send(ctx, struct{}{})
future := workflow.ExecuteChildWorkflow(chCtx, w, args...)
err := future.Get(ctx, nil)
s.statusCh.Send(ctx, err)
s.limit.Receive(ctx, nil)
})
}

func (s *BoundSelector) Wait() error {
for len(s.futures) > 0 {
s.selector.Select(s.ctx)
defer s.statusCh.Close()
defer s.limit.Close()

ferrors := make([]error, 0)
doneCount := 0

for doneCount < s.numFutures {
selector := workflow.NewSelector(s.ctx)
selector.AddReceive(s.statusCh, func(c workflow.ReceiveChannel, more bool) {
var err error
c.Receive(s.ctx, &err)
if err != nil {
ferrors = append(ferrors, err)
}
doneCount++
})
selector.Select(s.ctx)
}

if len(s.ferrors) > 0 {
return errors.Join(s.ferrors...)
if len(ferrors) > 0 {
return errors.Join(ferrors...)
}

return nil
Expand Down
66 changes: 33 additions & 33 deletions flow/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,51 @@ go 1.19

require (
cloud.google.com/go v0.110.7
cloud.google.com/go/bigquery v1.53.0
cloud.google.com/go/storage v1.31.0
cloud.google.com/go/bigquery v1.54.0
cloud.google.com/go/storage v1.32.0
github.com/Azure/azure-amqp-common-go/v4 v4.2.0
github.com/Azure/azure-event-hubs-go/v3 v3.6.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.1.1
github.com/aws/aws-sdk-go v1.44.317
github.com/google/uuid v1.3.0
github.com/aws/aws-sdk-go v1.44.332
github.com/google/uuid v1.3.1
github.com/hashicorp/go-multierror v1.1.1
github.com/jackc/pglogrepl v0.0.0-20230728225306-38e8a4e50913
github.com/jackc/pglogrepl v0.0.0-20230810221841-d0818e1fbef7
github.com/jackc/pgx/v5 v5.4.3
github.com/jmoiron/sqlx v1.3.5
github.com/joho/godotenv v1.5.1
github.com/lib/pq v1.10.9
github.com/linkedin/goavro/v2 v2.12.0
github.com/microsoft/go-mssqldb v1.5.0
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/prometheus/client_golang v1.16.0
github.com/sirupsen/logrus v1.9.3
github.com/snowflakedb/gosnowflake v1.6.23
github.com/snowflakedb/gosnowflake v1.6.24
github.com/stretchr/testify v1.8.4
github.com/uber-go/tally/v4 v4.1.7
github.com/urfave/cli/v2 v2.25.7
go.temporal.io/api v1.23.0
go.temporal.io/api v1.24.0
go.temporal.io/sdk v1.24.0
go.uber.org/automaxprocs v1.5.3
google.golang.org/api v0.134.0
google.golang.org/api v0.138.0
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
)

require (
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect
)

require (
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.1 // indirect
cloud.google.com/go/iam v1.1.2 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
github.com/Azure/go-amqp v1.0.1 // indirect
Expand All @@ -61,24 +61,24 @@ require (
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/apache/arrow/go/v12 v12.0.1 // indirect
github.com/apache/thrift v0.18.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.20.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.31 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 // indirect
github.com/aws/smithy-go v1.14.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.35 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.81 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
Expand All @@ -103,7 +103,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/s2a-go v0.1.5 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
Expand Down Expand Up @@ -133,14 +133,14 @@ require (
github.com/prometheus/procfs v0.11.1 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/stretchr/objx v0.5.1 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.temporal.io/sdk/contrib/tally v0.2.0
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
Expand All @@ -149,11 +149,11 @@ require (
golang.org/x/term v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.11.1 // indirect
golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Loading

0 comments on commit 2190dec

Please sign in to comment.