From 4acdba47575025d1b8fb4bb1d61931cbfffe4dc4 Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Mon, 17 Feb 2020 10:30:54 +0500 Subject: [PATCH 01/22] Feat: Ability to switch between multiple queues - Deletion based on which queue the message was read from --- v1/brokers/redis/redis.go | 4 +- v1/brokers/sqs/sqs.go | 72 ++++++++++++++--------- v1/brokers/sqs/sqs_export_test.go | 8 +-- v1/brokers/sqs/sqs_test.go | 98 +++++++++++++++++++++++++------ v1/worker.go | 11 ++++ 5 files changed, 142 insertions(+), 51 deletions(-) diff --git a/v1/brokers/redis/redis.go b/v1/brokers/redis/redis.go index c0a2d38b2..95db0e661 100644 --- a/v1/brokers/redis/redis.go +++ b/v1/brokers/redis/redis.go @@ -317,7 +317,9 @@ func (b *Broker) consumeOne(delivery []byte, taskProcessor iface.TaskProcessor) conn := b.open() defer conn.Close() - conn.Do("RPUSH", getQueue(b.GetConfig(), taskProcessor), delivery) + // Adjust routing key (this decides which queue the message will be send back to) + b.Broker.AdjustRoutingKey(signature) + conn.Do("RPUSH", signature.RoutingKey, delivery) return nil } diff --git a/v1/brokers/sqs/sqs.go b/v1/brokers/sqs/sqs.go index f811e16e5..eccecd974 100644 --- a/v1/brokers/sqs/sqs.go +++ b/v1/brokers/sqs/sqs.go @@ -35,7 +35,14 @@ type Broker struct { stopReceivingChan chan int sess *session.Session service sqsiface.SQSAPI - queueUrl *string +} + +// ReceivedMessages contains the queue name that the received message was fetched from so that we can delete the message later +// Since machinery can fetch messages from multiple source queues now, we need to either embed the source queue name inside the +// message signature (which can be prone to errors), or use this. +type ReceivedMessages struct { + Delivery *awssqs.ReceiveMessageOutput + queue *string } // New creates new Broker instance @@ -60,11 +67,7 @@ func New(cnf *config.Config) iface.Broker { // StartConsuming enters a loop and waits for incoming messages func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) - qURL := b.getQueueURL(taskProcessor) - //save it so that it can be used later when attempting to delete task - b.queueUrl = qURL - - deliveries := make(chan *awssqs.ReceiveMessageOutput, concurrency) + deliveries := make(chan *ReceivedMessages, concurrency) pool := make(chan struct{}, concurrency) // initialize worker pool with maxWorkers workers @@ -77,8 +80,7 @@ func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcess go func() { defer b.receivingWG.Done() - log.INFO.Printf("[*] Waiting for messages on queue: %s. To exit press CTRL+C\n", *qURL) - + log.INFO.Printf("[*] Waiting for messages on queue. To exit press CTRL+C\n") for { select { // A way to stop this goroutine from b.StopConsuming @@ -86,9 +88,10 @@ func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcess close(deliveries) return case <-pool: + qURL := b.getQueueURL(taskProcessor) output, err := b.receiveMessage(qURL) if err == nil && len(output.Messages) > 0 { - deliveries <- output + deliveries <- &ReceivedMessages{Delivery: output, queue:qURL} } else { //return back to pool right away @@ -135,7 +138,7 @@ func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error MsgInput := &awssqs.SendMessageInput{ MessageBody: aws.String(string(msg)), - QueueUrl: aws.String(b.GetConfig().Broker + "/" + signature.RoutingKey), + QueueUrl: b.queueToURL(signature.RoutingKey), } // if this is a fifo queue, there needs to be some additional parameters. @@ -178,7 +181,7 @@ func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error } // consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal -func (b *Broker) consume(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { +func (b *Broker) consume(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { errorsChan := make(chan error) @@ -194,7 +197,8 @@ func (b *Broker) consume(deliveries <-chan *awssqs.ReceiveMessageOutput, concurr } // consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS -func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error { +func (b *Broker) consumeOne(sqsReceivedMsgs *ReceivedMessages, taskProcessor iface.TaskProcessor) error { + delivery := sqsReceivedMsgs.Delivery if len(delivery.Messages) == 0 { log.ERROR.Printf("received an empty message, the delivery was %v", delivery) return errors.New("received empty message, the delivery is " + delivery.GoString()) @@ -206,7 +210,7 @@ func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor if err := decoder.Decode(sig); err != nil { log.ERROR.Printf("unmarshal error. the delivery is %v", delivery) // if the unmarshal fails, remove the delivery from the queue - if delErr := b.deleteOne(delivery); delErr != nil { + if delErr := b.deleteOne(sqsReceivedMsgs); delErr != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, delErr) } return err @@ -219,11 +223,16 @@ func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor // and leave the message in the queue if !b.IsTaskRegistered(sig.Name) { if sig.IgnoreWhenTaskNotRegistered { - b.deleteOne(delivery) + b.deleteOne(sqsReceivedMsgs) } return fmt.Errorf("task %s is not registered", sig.Name) } + if sqsReceivedMsgs.queue != nil { + queueName := b.urlToQueue(*(sqsReceivedMsgs.queue)) + sig.RoutingKey = *queueName + } + err := taskProcessor.Process(sig) if err != nil { // stop task deletion in case we want to send messages to dlq in sqs @@ -233,18 +242,19 @@ func (b *Broker) consumeOne(delivery *awssqs.ReceiveMessageOutput, taskProcessor return err } // Delete message after successfully consuming and processing the message - if err = b.deleteOne(delivery); err != nil { + if err = b.deleteOne(sqsReceivedMsgs); err != nil { log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err) } return err } // deleteOne is a method delete a delivery from AWS SQS -func (b *Broker) deleteOne(delivery *awssqs.ReceiveMessageOutput) error { - qURL := b.defaultQueueURL() +func (b *Broker) deleteOne(delivery *ReceivedMessages) error { + qURL := delivery.queue + _, err := b.service.DeleteMessage(&awssqs.DeleteMessageInput{ QueueUrl: qURL, - ReceiptHandle: delivery.Messages[0].ReceiptHandle, + ReceiptHandle: delivery.Delivery.Messages[0].ReceiptHandle, }) if err != nil { @@ -255,12 +265,7 @@ func (b *Broker) deleteOne(delivery *awssqs.ReceiveMessageOutput) error { // defaultQueueURL is a method returns the default queue url func (b *Broker) defaultQueueURL() *string { - if b.queueUrl != nil { - return b.queueUrl - } else { - return aws.String(b.GetConfig().Broker + "/" + b.GetConfig().DefaultQueue) - } - + return b.queueToURL(b.GetConfig().DefaultQueue) } // receiveMessage is a method receives a message from specified queue url @@ -302,7 +307,7 @@ func (b *Broker) initializePool(pool chan struct{}, concurrency int) { } // consumeDeliveries is a method consuming deliveries from deliveries channel -func (b *Broker) consumeDeliveries(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { +func (b *Broker) consumeDeliveries(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { select { case err := <-errorsChan: return false, err @@ -359,10 +364,19 @@ func (b *Broker) stopReceiving() { // getQueueURL is a method returns that returns queueURL first by checking if custom queue was set and usign it // otherwise using default queueName from config func (b *Broker) getQueueURL(taskProcessor iface.TaskProcessor) *string { - queueName := b.GetConfig().DefaultQueue - if taskProcessor.CustomQueue() != "" { - queueName = taskProcessor.CustomQueue() + if customQueue := taskProcessor.CustomQueue(); customQueue != "" { + return b.queueToURL(customQueue) } - return aws.String(b.GetConfig().Broker + "/" + queueName) + return b.defaultQueueURL() +} + +func (b *Broker) queueToURL(queue string) *string { + return aws.String(b.GetConfig().Broker + "/" + queue) +} + +func (b *Broker) urlToQueue(queue string) *string { + parts := strings.Split(queue, "/") + queueName := parts[len(parts) - 1] + return &queueName } diff --git a/v1/brokers/sqs/sqs_export_test.go b/v1/brokers/sqs/sqs_export_test.go index 25b0348b2..8a2d822d2 100644 --- a/v1/brokers/sqs/sqs_export_test.go +++ b/v1/brokers/sqs/sqs_export_test.go @@ -126,15 +126,15 @@ func init() { } } -func (b *Broker) ConsumeForTest(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { +func (b *Broker) ConsumeForTest(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { return b.consume(deliveries, concurrency, taskProcessor, pool) } -func (b *Broker) ConsumeOneForTest(delivery *awssqs.ReceiveMessageOutput, taskProcessor iface.TaskProcessor) error { +func (b *Broker) ConsumeOneForTest(delivery *ReceivedMessages, taskProcessor iface.TaskProcessor) error { return b.consumeOne(delivery, taskProcessor) } -func (b *Broker) DeleteOneForTest(delivery *awssqs.ReceiveMessageOutput) error { +func (b *Broker) DeleteOneForTest(delivery *ReceivedMessages) error { return b.deleteOne(delivery) } @@ -150,7 +150,7 @@ func (b *Broker) InitializePoolForTest(pool chan struct{}, concurrency int) { b.initializePool(pool, concurrency) } -func (b *Broker) ConsumeDeliveriesForTest(deliveries <-chan *awssqs.ReceiveMessageOutput, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { +func (b *Broker) ConsumeDeliveriesForTest(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { return b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) } diff --git a/v1/brokers/sqs/sqs_test.go b/v1/brokers/sqs/sqs_test.go index f066ef6d7..94032041f 100644 --- a/v1/brokers/sqs/sqs_test.go +++ b/v1/brokers/sqs/sqs_test.go @@ -92,10 +92,11 @@ func TestPrivateFunc_consume(t *testing.T) { } pool := make(chan struct{}, 0) wk := server1.NewWorker("sms_worker", 0) - deliveries := make(chan *awssqs.ReceiveMessageOutput) + deliveries := make(chan *sqs.ReceivedMessages) outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{} - go func() { deliveries <- &outputCopy }() + receivedMsg := sqs.ReceivedMessages{Delivery: &outputCopy} + go func() { deliveries <- &receivedMsg }() // an infinite loop will be executed only when there is no error err = testAWSSQSBroker.ConsumeForTest(deliveries, 0, wk, pool) @@ -109,12 +110,13 @@ func TestPrivateFunc_consumeOne(t *testing.T) { t.Fatal(err) } wk := server1.NewWorker("sms_worker", 0) - err = testAWSSQSBroker.ConsumeOneForTest(receiveMessageOutput, wk) - assert.NotNil(t, err) - outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{} - err = testAWSSQSBroker.ConsumeOneForTest(&outputCopy, wk) + receivedMsg := sqs.ReceivedMessages{Delivery: &outputCopy} + err = testAWSSQSBroker.ConsumeOneForTest(&receivedMsg, wk) + assert.NotNil(t, err) + + err = testAWSSQSBroker.ConsumeOneForTest(&receivedMsg, wk) assert.NotNil(t, err) outputCopy.Messages = []*awssqs.Message{ @@ -122,7 +124,8 @@ func TestPrivateFunc_consumeOne(t *testing.T) { Body: aws.String("foo message"), }, } - err = testAWSSQSBroker.ConsumeOneForTest(&outputCopy, wk) + receivedMsg = sqs.ReceivedMessages{Delivery: &outputCopy} + err = testAWSSQSBroker.ConsumeOneForTest(&receivedMsg, wk) assert.NotNil(t, err) } @@ -173,13 +176,16 @@ func TestPrivateFunc_consumeDeliveries(t *testing.T) { concurrency := 0 pool := make(chan struct{}, concurrency) errorsChan := make(chan error) - deliveries := make(chan *awssqs.ReceiveMessageOutput) + deliveries := make(chan *sqs.ReceivedMessages) server1, err := machinery.NewServer(cnf) if err != nil { t.Fatal(err) } wk := server1.NewWorker("sms_worker", 0) - go func() { deliveries <- receiveMessageOutput }() + outputCopy := *receiveMessageOutput + outputCopy.Messages = []*awssqs.Message{} + receivedMsg := sqs.ReceivedMessages{Delivery:&outputCopy} + go func() { deliveries <- &receivedMsg }() whetherContinue, err := testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) assert.True(t, whetherContinue) assert.Nil(t, err) @@ -194,9 +200,10 @@ func TestPrivateFunc_consumeDeliveries(t *testing.T) { assert.False(t, whetherContinue) assert.Nil(t, err) - outputCopy := *receiveMessageOutput + outputCopy = *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{} - go func() { deliveries <- &outputCopy }() + receivedMsg = sqs.ReceivedMessages{Delivery:&outputCopy} + go func() { deliveries <- &receivedMsg }() whetherContinue, err = testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) e := <-errorsChan assert.True(t, whetherContinue) @@ -204,6 +211,9 @@ func TestPrivateFunc_consumeDeliveries(t *testing.T) { assert.Nil(t, err) // using a wait group and a channel to fix the racing problem + outputCopy = *receiveMessageOutput + outputCopy.Messages = []*awssqs.Message{} + receivedMsg = sqs.ReceivedMessages{Delivery:&outputCopy} var wg sync.WaitGroup wg.Add(1) nextStep := make(chan bool, 1) @@ -211,7 +221,7 @@ func TestPrivateFunc_consumeDeliveries(t *testing.T) { defer wg.Done() // nextStep <- true runs after defer wg.Done(), to make sure the next go routine runs after this go routine nextStep <- true - deliveries <- receiveMessageOutput + deliveries <- &receivedMsg }() if <-nextStep { // <-pool will block the routine in the following steps, so pool <- struct{}{} will be executed for sure @@ -226,10 +236,18 @@ func TestPrivateFunc_consumeDeliveries(t *testing.T) { } func TestPrivateFunc_deleteOne(t *testing.T) { - err := testAWSSQSBroker.DeleteOneForTest(receiveMessageOutput) + outputCopy := *receiveMessageOutput + outputCopy.Messages = []*awssqs.Message{ + { + Body: aws.String("foo message"), + ReceiptHandle: aws.String("receipthandle"), + }, + } + receivedMsg := sqs.ReceivedMessages{Delivery: &outputCopy} + err := testAWSSQSBroker.DeleteOneForTest(&receivedMsg) assert.Nil(t, err) - err = errAWSSQSBroker.DeleteOneForTest(receiveMessageOutput) + err = errAWSSQSBroker.DeleteOneForTest(&receivedMsg) assert.NotNil(t, err) } @@ -275,7 +293,7 @@ func TestPrivateFunc_consumeWithConcurrency(t *testing.T) { pool := make(chan struct{}, 1) pool <- struct{}{} wk := server1.NewWorker("sms_worker", 1) - deliveries := make(chan *awssqs.ReceiveMessageOutput) + deliveries := make(chan *sqs.ReceivedMessages) outputCopy := *receiveMessageOutput outputCopy.Messages = []*awssqs.Message{ { @@ -283,9 +301,9 @@ func TestPrivateFunc_consumeWithConcurrency(t *testing.T) { Body: aws.String(msg), }, } - + receivedMsg := sqs.ReceivedMessages{Delivery:&outputCopy} go func() { - deliveries <- &outputCopy + deliveries <- &receivedMsg }() @@ -302,3 +320,49 @@ func TestPrivateFunc_consumeWithConcurrency(t *testing.T) { t.Fatal("task not processed in 10 seconds") } } + +type roundRobinQueues struct { + queues []string + currentIndex int +} + +func NewRoundRobinQueues(queues []string) *roundRobinQueues { + return &roundRobinQueues{ + queues: queues, + currentIndex: -1, + } +} + +func (r *roundRobinQueues) Peek() string { + return r.queues[r.currentIndex] +} + +func (r *roundRobinQueues) Next() string { + r.currentIndex += 1 + if r.currentIndex >= len(r.queues) { + r.currentIndex = 0 + } + + q := r.queues[r.currentIndex] + return q +} + +func TestPrivateFunc_consumeWithRoundRobinQueues(t *testing.T) { + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + + w := server1.NewWorker("test-worker", 0) + + // Assigning a getQueueHandler to `Next` method of roundRobinQueues + rr := NewRoundRobinQueues([]string{"custom-queue-0", "custom-queue-1", "custom-queue-2", "custom-queue-3"}) + w.SetGetQueueHandler(rr.Next) + + for i := 0; i < 5; i++ { + // the queue url of the broker should match the current queue url of roundRobin + // and thus queues are being utilized in round-robin fashion + qURL := testAWSSQSBroker.GetQueueURLForTest(w) + assert.Equal(t, qURL, testAWSSQSBroker.GetCustomQueueURL(rr.Peek())) + } +} diff --git a/v1/worker.go b/v1/worker.go index d09445634..7def12631 100644 --- a/v1/worker.go +++ b/v1/worker.go @@ -26,6 +26,7 @@ type Worker struct { errorHandler func(err error) preTaskHandler func(*tasks.Signature) postTaskHandler func(*tasks.Signature) + getQueueHandler func() string } var ( @@ -116,6 +117,11 @@ func (worker *Worker) LaunchAsync(errorsChan chan<- error) { // CustomQueue returns Custom Queue of the running worker process func (worker *Worker) CustomQueue() string { + // if the handler is defined, use that to fetch the queue name + if worker.getQueueHandler != nil { + return worker.getQueueHandler() + } + return worker.Queue } @@ -399,6 +405,11 @@ func (worker *Worker) SetPostTaskHandler(handler func(*tasks.Signature)) { worker.postTaskHandler = handler } +//SetGetQueueHandler sets a get queue handler to fetch queue name from +func (worker *Worker) SetGetQueueHandler(handler func() string) { + worker.getQueueHandler = handler +} + //GetServer returns server func (worker *Worker) GetServer() *Server { return worker.server From 920e7add9923e6cfdfd661d01f4c0b870d1a5f0a Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Tue, 1 Sep 2020 12:07:57 +0500 Subject: [PATCH 02/22] Feat: add redis-dlq broker --- go.mod | 2 +- go.sum | 20 ++ v1/brokers/redis/goredis-dlq.go | 473 ++++++++++++++++++++++++++++++++ v1/config/config.go | 4 + v1/factories.go | 12 + 5 files changed, 510 insertions(+), 1 deletion(-) create mode 100644 v1/brokers/redis/goredis-dlq.go diff --git a/go.mod b/go.mod index e3f0967d8..bdf17f0d2 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/RichardKnop/redsync v1.2.0 github.com/aws/aws-sdk-go v1.25.8 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b + github.com/btcsuite/btcutil v1.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/go-redis/redis v6.15.6+incompatible github.com/go-stack/stack v1.8.0 // indirect @@ -29,7 +30,6 @@ require ( github.com/xdg/stringprep v1.0.0 // indirect go.mongodb.org/mongo-driver v1.1.2 go.opencensus.io v0.22.1 // indirect - golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc // indirect golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect diff --git a/go.sum b/go.sum index 834086303..cceda8db7 100644 --- a/go.sum +++ b/go.sum @@ -22,15 +22,27 @@ github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae h1:DcFpTQBYQ9C github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae/go.mod h1:rJJ84PyA/Wlmw1hO+xTzV2wsSUon6J5ktg0g8BF2PuU= github.com/RichardKnop/redsync v1.2.0 h1:gK35hR3zZkQigHKm8wOGb9MpJ9BsrW6MzxezwjTcHP0= github.com/RichardKnop/redsync v1.2.0/go.mod h1:9b8nBGAX3bE2uCfJGSnsDvF23mKyHTZzmvmj5FH3Tp0= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs= github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2 h1:9iZ1Terx9fMIOtq1VrwdqfsATL9MC2l8ZrUY6YZ2uts= +github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -79,8 +91,10 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= @@ -88,14 +102,17 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.0 h1:XaTQmdKecIbwNHpzOIy0XMoEG5bmv/n0OVyaF1NKUdo= github.com/onsi/ginkgo v1.10.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -132,11 +149,14 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go new file mode 100644 index 000000000..2c04d3c91 --- /dev/null +++ b/v1/brokers/redis/goredis-dlq.go @@ -0,0 +1,473 @@ +package redis + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/RichardKnop/machinery/v1/brokers/errs" + "github.com/RichardKnop/machinery/v1/brokers/iface" + "github.com/RichardKnop/machinery/v1/common" + "github.com/RichardKnop/machinery/v1/config" + "github.com/RichardKnop/machinery/v1/log" + "github.com/RichardKnop/machinery/v1/tasks" + "github.com/RichardKnop/redsync" + "github.com/btcsuite/btcutil/base58" + "github.com/go-redis/redis" +) + +const ( + messageVisibilitySet = "message-visibility-set" + hSetMessageKey = "message" + hSetQueueKey = "queue" + taskPrefix = "task_%s" +) + +// BrokerGR_DLQ represents a Redis broker using go-redis, with enhancements for DLQ support +type BrokerGR_DLQ struct { + common.Broker + rclient redis.UniversalClient + consumingWG sync.WaitGroup // wait group to make sure whole consumption completes + processingWG sync.WaitGroup // use wait group to make sure task processing completes + delayedWG sync.WaitGroup + // If set, path to a socket file overrides hostname + socketPath string + redsync *redsync.Redsync + redisOnce sync.Once +} + +// NewGR_DLQ creates new Broker instance +func NewGR_DLQ(cnf *config.Config, addrs []string, db int) iface.Broker { + b := &BrokerGR_DLQ{Broker: common.NewBroker(cnf)} + for i, addr := range addrs { + //go-redis URI can't handle db at the end of URI + addrs[i] = strings.Split(addr, "/")[0] + } + + var password string + parts := strings.Split(addrs[0], "@") + if len(parts) == 2 { + // with passwrod + password = parts[0] + addrs[0] = parts[1] + } + + ropt := &redis.UniversalOptions{ + Addrs: addrs, + DB: db, + Password: password, + } + if cnf.Redis != nil { + ropt.MasterName = cnf.Redis.MasterName + } + + b.rclient = redis.NewUniversalClient(ropt) + + if cnf.Redis.DelayedTasksKey != "" { + redisDelayedTasksKey = cnf.Redis.DelayedTasksKey + } + return b +} + +// StartConsuming enters a loop and waits for incoming messages +func (b *BrokerGR_DLQ) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { + b.consumingWG.Add(1) + defer b.consumingWG.Done() + + if concurrency < 1 { + concurrency = 1 + } + + b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) + + // Ping the server to make sure connection is live + _, err := b.rclient.Ping().Result() + if err != nil { + b.GetRetryFunc()(b.GetRetryStopChan()) + + // Return err if retry is still true. + // If retry is false, broker.StopConsuming() has been called and + // therefore Redis might have been stopped. Return nil exit + // StartConsuming() + if b.GetRetry() { + return b.GetRetry(), err + } + return b.GetRetry(), errs.ErrConsumerStopped + } + + // Channel to which we will push tasks ready for processing by worker + deliveries := make(chan []byte, concurrency) + pool := make(chan struct{}, concurrency) + + // initialize worker pool with maxWorkers workers + for i := 0; i < concurrency; i++ { + pool <- struct{}{} + } + + // A receiving goroutine keeps popping messages from the queue by BLPOP + // If the message is valid and can be unmarshaled into a proper structure + // we send it to the deliveries channel + go func() { + + log.INFO.Print("[*] Waiting for messages. To exit press CTRL+C") + + for { + select { + // A way to stop this goroutine from b.StopConsuming + case <-b.GetStopChan(): + close(deliveries) + return + case <-pool: + task, _ := b.nextTask(getQueueGR(b.GetConfig(), taskProcessor)) + //TODO: should this error be ignored? + if len(task) > 0 { + deliveries <- task + } + + pool <- struct{}{} + } + } + }() + + // A goroutine to watch for delayed tasks and push them to deliveries + // channel for consumption by the worker + b.delayedWG.Add(1) + go func() { + defer b.delayedWG.Done() + + for { + select { + // A way to stop this goroutine from b.StopConsuming + case <-b.GetStopChan(): + return + default: + task, err := b.nextDelayedTask(redisDelayedTasksKey) + if err != nil { + continue + } + + signature := new(tasks.Signature) + decoder := json.NewDecoder(bytes.NewReader(task)) + decoder.UseNumber() + if err := decoder.Decode(signature); err != nil { + log.ERROR.Print(errs.NewErrCouldNotUnmarshaTaskSignature(task, err)) + } + + if err := b.Publish(context.Background(), signature); err != nil { + log.ERROR.Print(err) + } + } + } + }() + + if err := b.consume(deliveries, concurrency, taskProcessor); err != nil { + return b.GetRetry(), err + } + + // Waiting for any tasks being processed to finish + b.processingWG.Wait() + + return b.GetRetry(), nil +} + +// StopConsuming quits the loop +func (b *BrokerGR_DLQ) StopConsuming() { + b.Broker.StopConsuming() + // Waiting for the delayed tasks goroutine to have stopped + b.delayedWG.Wait() + // Waiting for consumption to finish + b.consumingWG.Wait() + + b.rclient.Close() +} + +// Publish places a new message on the default queue +func (b *BrokerGR_DLQ) Publish(ctx context.Context, signature *tasks.Signature) error { + // Adjust routing key (this decides which queue the message will be published to) + b.Broker.AdjustRoutingKey(signature) + + msg, err := json.Marshal(signature) + if err != nil { + return fmt.Errorf("JSON marshal error: %s", err) + } + + // Check the ETA signature field, if it is set and it is in the future, + // delay the task + if signature.ETA != nil { + now := time.Now().UTC() + + if signature.ETA.After(now) { + score := signature.ETA.UnixNano() + err = b.rclient.ZAdd(redisDelayedTasksKey, redis.Z{Score: float64(score), Member: msg}).Err() + return err + } + } + + err = b.rclient.RPush(signature.RoutingKey, msg).Err() + return err +} + +// GetPendingTasks returns a slice of task signatures waiting in the queue +func (b *BrokerGR_DLQ) GetPendingTasks(queue string) ([]*tasks.Signature, error) { + + if queue == "" { + queue = b.GetConfig().DefaultQueue + } + results, err := b.rclient.LRange(queue, 0, -1).Result() + if err != nil { + return nil, err + } + + taskSignatures := make([]*tasks.Signature, len(results)) + for i, result := range results { + signature := new(tasks.Signature) + decoder := json.NewDecoder(strings.NewReader(result)) + decoder.UseNumber() + if err := decoder.Decode(signature); err != nil { + return nil, err + } + taskSignatures[i] = signature + } + return taskSignatures, nil +} + +// GetDelayedTasks returns a slice of task signatures that are scheduled, but not yet in the queue +func (b *BrokerGR_DLQ) GetDelayedTasks() ([]*tasks.Signature, error) { + results, err := b.rclient.ZRange(redisDelayedTasksKey, 0, -1).Result() + if err != nil { + return nil, err + } + + taskSignatures := make([]*tasks.Signature, len(results)) + for i, result := range results { + signature := new(tasks.Signature) + decoder := json.NewDecoder(strings.NewReader(result)) + decoder.UseNumber() + if err := decoder.Decode(signature); err != nil { + return nil, err + } + taskSignatures[i] = signature + } + return taskSignatures, nil +} + +// consume takes delivered messages from the channel and manages a worker pool +// to process tasks concurrently +func (b *BrokerGR_DLQ) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor) error { + errorsChan := make(chan error, concurrency*2) + pool := make(chan struct{}, concurrency) + + // init pool for Worker tasks execution, as many slots as Worker concurrency param + go func() { + for i := 0; i < concurrency; i++ { + pool <- struct{}{} + } + }() + + for { + select { + case err := <-errorsChan: + return err + case d, open := <-deliveries: + if !open { + return nil + } + if concurrency > 0 { + // get execution slot from pool (blocks until one is available) + <-pool + } + + b.processingWG.Add(1) + + // Consume the task inside a goroutine so multiple tasks + // can be processed concurrently + go func() { + if err := b.consumeOne(d, taskProcessor); err != nil { + errorsChan <- err + } + + b.processingWG.Done() + + if concurrency > 0 { + // give slot back to pool + pool <- struct{}{} + } + }() + } + } +} + +// consumeOne processes a single message using TaskProcessor +func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProcessor) error { + signature := new(tasks.Signature) + decoder := json.NewDecoder(bytes.NewReader(delivery)) + decoder.UseNumber() + if err := decoder.Decode(signature); err != nil { + return errs.NewErrCouldNotUnmarshaTaskSignature(delivery, err) + } + // propagating hash UUID for possible application usage, for example, refreshing visibility + gHashByte := sha256.Sum256(delivery) + gHash := fmt.Sprintf(taskPrefix, base58.Encode(gHashByte[:sha256.Size])) + signature.UUID = gHash + + // If the task is not registered, we requeue it, + // there might be different workers for processing specific tasks + if !b.IsTaskRegistered(signature.Name) { + log.INFO.Printf("Task not registered with this worker. Requeing message: %s", delivery) + + b.rclient.RPush(getQueueGR(b.GetConfig(), taskProcessor), delivery) + return nil + } + + log.DEBUG.Printf("Received new message: %s", delivery) + + if err := taskProcessor.Process(signature); err != nil { + return err + } + + if err := b.deleteOne(signature); err != nil { + log.ERROR.Printf("error when deleting the delivery. Error=%s", err) + return err + } + + return nil +} + +// nextTask pops next available task from the default queue +func (b *BrokerGR_DLQ) nextTask(queue string) (result []byte, err error) { + + pollPeriodMilliseconds := 1000 // default poll period for normal tasks + if b.GetConfig().Redis != nil { + configuredPollPeriod := b.GetConfig().Redis.NormalTasksPollPeriod + if configuredPollPeriod > 0 { + pollPeriodMilliseconds = configuredPollPeriod + } + } + pollPeriod := time.Duration(pollPeriodMilliseconds) * time.Millisecond + watchFunc := func(tx *redis.Tx) error { + items, err := b.rclient.BLPop(pollPeriod, queue).Result() + if err != nil { + return err + } + result = []byte(items[1]) + + // items[0] - the name of the key where an element was popped + // items[1] - the value of the popped element + if len(items) != 2 { + return redis.Nil + } + gHashByte := sha256.Sum256([]byte(items[1])) + gHash := fmt.Sprintf(taskPrefix, base58.Encode(gHashByte[:sha256.Size])) + + fields := map[string]interface{}{ + hSetMessageKey: items[1], + hSetQueueKey: queue, + } + if err := b.rclient.HMSet(gHash, fields).Err(); err != nil { + return err + } + + z := redis.Z{Score: float64(time.Now().Add(1 * time.Minute).Unix()), Member: gHash} + return b.rclient.ZAdd(messageVisibilitySet, z).Err() + } + + err = b.rclient.Watch(watchFunc, queue) + if err != nil { + return nil, err + } + + return result, nil +} + +// nextDelayedTask pops a value from the ZSET key using WATCH/MULTI/EXEC commands. +func (b *BrokerGR_DLQ) nextDelayedTask(key string) (result []byte, err error) { + + //pipe := b.rclient.Pipeline() + // + //defer func() { + // // Return connection to normal state on error. + // // https://redis.io/commands/discard + // if err != nil { + // pipe.Discard() + // } + //}() + + var ( + items []string + reply interface{} + ) + + pollPeriod := 500 // default poll period for delayed tasks + if b.GetConfig().Redis != nil { + configuredPollPeriod := b.GetConfig().Redis.DelayedTasksPollPeriod + // the default period is 0, which bombards redis with requests, despite + // our intention of doing the opposite + if configuredPollPeriod > 0 { + pollPeriod = configuredPollPeriod + } + } + + for { + // Space out queries to ZSET so we don't bombard redis + // server with relentless ZRANGEBYSCOREs + time.Sleep(time.Duration(pollPeriod) * time.Millisecond) + watchFunc := func(tx *redis.Tx) error { + + now := time.Now().UTC().UnixNano() + + // https://redis.io/commands/zrangebyscore + items, err = tx.ZRevRangeByScore(key, redis.ZRangeBy{ + Min: "0", Max: strconv.FormatInt(now, 10), Offset: 0, Count: 1, + }).Result() + if err != nil { + return err + } + if len(items) != 1 { + return redis.Nil + } + + return nil + + } + if err = b.rclient.Watch(watchFunc, key); err != nil { + return + } + + txpipe := b.rclient.TxPipeline() + txpipe.ZRem(key, items[0]) + reply, err = txpipe.Exec() + if err != nil { + return + } + + if reply != nil { + result = []byte(items[0]) + break + } + } + + return +} + +// deleteOne is a method to delete a redis message from the message visibility set as well as its temporary HSET +func (b *BrokerGR_DLQ) deleteOne(signature *tasks.Signature) error { + + gHash := signature.UUID + watchFunc := func(tx *redis.Tx) error { + if err := b.rclient.Del(gHash).Err(); err != nil { + return err + } + if err := b.rclient.ZRem(messageVisibilitySet, gHash).Err(); err != nil { + return err + } + return nil + } + // not watching gHash, because do not want transaction to cancel in any case + return b.rclient.Watch(watchFunc) +} diff --git a/v1/config/config.go b/v1/config/config.go index b2ff68886..c0147d3e1 100644 --- a/v1/config/config.go +++ b/v1/config/config.go @@ -137,6 +137,10 @@ type RedisConfig struct { // Default: 15 ConnectTimeout int `yaml:"connect_timeout" envconfig:"REDIS_CONNECT_TIMEOUT"` + // VisibilityTimeout used in redis-dlq broker + // default to nil to use the overall visibility timeout for redis messages + VisibilityTimeout *int64 `yaml:"visibility_timeout" envconfig:"REDIS_VISIBILITY_TIMEOUT"` + // NormalTasksPollPeriod specifies the period in milliseconds when polling redis for normal tasks // Default: 1000 NormalTasksPollPeriod int `yaml:"normal_tasks_poll_period" envconfig:"REDIS_NORMAL_TASKS_POLL_PERIOD"` diff --git a/v1/factories.go b/v1/factories.go index 9dcebca30..0416c1751 100644 --- a/v1/factories.go +++ b/v1/factories.go @@ -58,6 +58,18 @@ func BrokerFactory(cnf *config.Config) (brokeriface.Broker, error) { } } + if strings.HasPrefix(cnf.Broker, "redis+dlq://") { + parts := strings.Split(cnf.Broker, "redis+dlq://") + if len(parts) != 2 { + return nil, fmt.Errorf( + "Redis DLQ broker connection string should be in format redis://host:port, instead got %s", + cnf.Broker, + ) + } + brokers := strings.Split(parts[1], ",") + return redisbroker.NewGR_DLQ(cnf, brokers, 0), nil + } + if strings.HasPrefix(cnf.Broker, "redis+socket://") { redisSocket, redisPassword, redisDB, err := ParseRedisSocketURL(cnf.Broker) if err != nil { From 6f081ae910415c84fe1246cd8dd365eab370b056 Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Sat, 10 Oct 2020 05:58:02 +0500 Subject: [PATCH 03/22] logging and error fixes Since we modify the signature with the message hash, we shouldn't print the unmodified "delivery", but the modified "signature" --- v1/brokers/redis/goredis-dlq.go | 4 ++-- v1/factories.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index 2c04d3c91..74db0bd22 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -319,13 +319,13 @@ func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProce // If the task is not registered, we requeue it, // there might be different workers for processing specific tasks if !b.IsTaskRegistered(signature.Name) { - log.INFO.Printf("Task not registered with this worker. Requeing message: %s", delivery) + log.INFO.Printf("Task not registered with this worker. Requeing message: %+v", signature) b.rclient.RPush(getQueueGR(b.GetConfig(), taskProcessor), delivery) return nil } - log.DEBUG.Printf("Received new message: %s", delivery) + log.DEBUG.Printf("Received new message: %+v", signature) if err := taskProcessor.Process(signature); err != nil { return err diff --git a/v1/factories.go b/v1/factories.go index 0416c1751..b5d1a84d9 100644 --- a/v1/factories.go +++ b/v1/factories.go @@ -62,7 +62,7 @@ func BrokerFactory(cnf *config.Config) (brokeriface.Broker, error) { parts := strings.Split(cnf.Broker, "redis+dlq://") if len(parts) != 2 { return nil, fmt.Errorf( - "Redis DLQ broker connection string should be in format redis://host:port, instead got %s", + "Redis DLQ broker connection string should be in format redis+dlq://host:port, instead got %s", cnf.Broker, ) } From 8b8940c542989af45affd944320ce7cfb079f79f Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Sat, 10 Oct 2020 05:58:34 +0500 Subject: [PATCH 04/22] remove separate worker pool buffer in redis-dlq broker Previously, one channel used to hold tasks in-memory, and the tasks waited to be processed until the machinery worker had was able to process them. In the go-redis-dlq broker, visibilityTimeout starts immediately after the task has been fetched from redis, which meant the visibilityTimeout of the task was draining even if it was waiting to be processed by machinery. Removed the separate pool for worker instances, bringing it inline to the SQS broker, where there is only one pool of task workers, and messages are only fetched from redis if a worker is present to process them immediately. --- v1/brokers/redis/goredis-dlq.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index 74db0bd22..8459151cf 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -128,9 +128,9 @@ func (b *BrokerGR_DLQ) StartConsuming(consumerTag string, concurrency int, taskP //TODO: should this error be ignored? if len(task) > 0 { deliveries <- task + } else { + pool <- struct{}{} } - - pool <- struct{}{} } } }() @@ -166,7 +166,7 @@ func (b *BrokerGR_DLQ) StartConsuming(consumerTag string, concurrency int, taskP } }() - if err := b.consume(deliveries, concurrency, taskProcessor); err != nil { + if err := b.consume(deliveries, concurrency, taskProcessor, pool); err != nil { return b.GetRetry(), err } @@ -259,16 +259,8 @@ func (b *BrokerGR_DLQ) GetDelayedTasks() ([]*tasks.Signature, error) { // consume takes delivered messages from the channel and manages a worker pool // to process tasks concurrently -func (b *BrokerGR_DLQ) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor) error { - errorsChan := make(chan error, concurrency*2) - pool := make(chan struct{}, concurrency) - - // init pool for Worker tasks execution, as many slots as Worker concurrency param - go func() { - for i := 0; i < concurrency; i++ { - pool <- struct{}{} - } - }() +func (b *BrokerGR_DLQ) consume(deliveries <-chan []byte, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { + errorsChan := make(chan error) for { select { @@ -278,10 +270,6 @@ func (b *BrokerGR_DLQ) consume(deliveries <-chan []byte, concurrency int, taskPr if !open { return nil } - if concurrency > 0 { - // get execution slot from pool (blocks until one is available) - <-pool - } b.processingWG.Add(1) From 843350375ecaaf72762917a9edc1783783c759fc Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Mon, 12 Oct 2020 18:11:58 +0500 Subject: [PATCH 05/22] fix data-race issues in redis-dlq broker 1. Tasks were being duplicated among different workers when consuming form the delayed_tasks ZSET. 2. Task processing was failing because HSET entries were not being set after elements were popped from queue, because of Watch translation failing. Fixed this by first peeking into list by using LRANGE, and then removing from the queue only if all other data structures have been set. 3. Stop error propogation into broker if DLQ is enabled in task signature. --- v1/brokers/redis/goredis-dlq.go | 70 +++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index 8459151cf..a8bc33662 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -316,6 +316,10 @@ func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProce log.DEBUG.Printf("Received new message: %+v", signature) if err := taskProcessor.Process(signature); err != nil { + // stop task deletion in case we want to send messages to dlq in redis + if err == errs.ErrStopTaskDeletion { + return nil + } return err } @@ -338,35 +342,55 @@ func (b *BrokerGR_DLQ) nextTask(queue string) (result []byte, err error) { } } pollPeriod := time.Duration(pollPeriodMilliseconds) * time.Millisecond + visibilityTimeout := *b.GetConfig().Redis.VisibilityTimeout + if visibilityTimeout <= 0 { + visibilityTimeout = 60 + } watchFunc := func(tx *redis.Tx) error { - items, err := b.rclient.BLPop(pollPeriod, queue).Result() + items, err := tx.LRange(queue, 0, 0).Result() if err != nil { return err } - result = []byte(items[1]) - // items[0] - the name of the key where an element was popped // items[1] - the value of the popped element - if len(items) != 2 { + if len(items) != 1 { return redis.Nil } - gHashByte := sha256.Sum256([]byte(items[1])) + gHashByte := sha256.Sum256([]byte(items[0])) gHash := fmt.Sprintf(taskPrefix, base58.Encode(gHashByte[:sha256.Size])) fields := map[string]interface{}{ - hSetMessageKey: items[1], + hSetMessageKey: items[0], hSetQueueKey: queue, } - if err := b.rclient.HMSet(gHash, fields).Err(); err != nil { + z := redis.Z{Score: float64(time.Now().Add(time.Duration(visibilityTimeout) * time.Second).Unix()), Member: gHash} + _, err = tx.TxPipelined(func(pipe redis.Pipeliner) error { + if err := pipe.HMSet(gHash, fields).Err(); err != nil { + return err + } + if err := pipe.ZAdd(messageVisibilitySet, z).Err(); err != nil { + return err + } + if err := pipe.LRem(queue, 1, items[0]).Err(); err != nil { + return err + } + return nil + }) + + if err != nil { return err } - - z := redis.Z{Score: float64(time.Now().Add(1 * time.Minute).Unix()), Member: gHash} - return b.rclient.ZAdd(messageVisibilitySet, z).Err() + result = []byte(items[0]) + return nil } err = b.rclient.Watch(watchFunc, queue) if err != nil { + if err == redis.Nil { + // if no keys found then need to delay to stop constant bombarding + time.Sleep(pollPeriod) + } + return nil, err } @@ -386,10 +410,7 @@ func (b *BrokerGR_DLQ) nextDelayedTask(key string) (result []byte, err error) { // } //}() - var ( - items []string - reply interface{} - ) + var items []string pollPeriod := 500 // default poll period for delayed tasks if b.GetConfig().Redis != nil { @@ -419,25 +440,16 @@ func (b *BrokerGR_DLQ) nextDelayedTask(key string) (result []byte, err error) { if len(items) != 1 { return redis.Nil } - - return nil - + _, err = tx.TxPipelined(func(pipe redis.Pipeliner) error { + return pipe.ZRem(key, items[0]).Err() + }) + return err } if err = b.rclient.Watch(watchFunc, key); err != nil { return } - - txpipe := b.rclient.TxPipeline() - txpipe.ZRem(key, items[0]) - reply, err = txpipe.Exec() - if err != nil { - return - } - - if reply != nil { - result = []byte(items[0]) - break - } + result = []byte(items[0]) + break } return From 8e6b9be528177385f44277d251d4917136bd018c Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Wed, 18 Nov 2020 01:02:27 +0500 Subject: [PATCH 06/22] Fix: use redis db if provided in config URL --- go.mod | 2 +- v1/brokers/redis/goredis-dlq.go | 15 +--- v1/factories.go | 82 +++++++++++++++++++--- v1/factories_test.go | 117 +++++++++++++++++++++++++++++++- 4 files changed, 193 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index bdf17f0d2..d2536db6b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/RichardKnop/redsync v1.2.0 github.com/aws/aws-sdk-go v1.25.8 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b - github.com/btcsuite/btcutil v1.0.2 // indirect + github.com/btcsuite/btcutil v1.0.2 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/go-redis/redis v6.15.6+incompatible github.com/go-stack/stack v1.8.0 // indirect diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index a8bc33662..a07a6740a 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -43,20 +43,8 @@ type BrokerGR_DLQ struct { } // NewGR_DLQ creates new Broker instance -func NewGR_DLQ(cnf *config.Config, addrs []string, db int) iface.Broker { +func NewGR_DLQ(cnf *config.Config, addrs []string, password string, db int) iface.Broker { b := &BrokerGR_DLQ{Broker: common.NewBroker(cnf)} - for i, addr := range addrs { - //go-redis URI can't handle db at the end of URI - addrs[i] = strings.Split(addr, "/")[0] - } - - var password string - parts := strings.Split(addrs[0], "@") - if len(parts) == 2 { - // with passwrod - password = parts[0] - addrs[0] = parts[1] - } ropt := &redis.UniversalOptions{ Addrs: addrs, @@ -64,6 +52,7 @@ func NewGR_DLQ(cnf *config.Config, addrs []string, db int) iface.Broker { Password: password, } if cnf.Redis != nil { + // if we're specifying MasterName here, then we'll always connect to db 0, since provided db is ignored in cluster mode ropt.MasterName = cnf.Redis.MasterName } diff --git a/v1/factories.go b/v1/factories.go index b5d1a84d9..60020c87f 100644 --- a/v1/factories.go +++ b/v1/factories.go @@ -59,15 +59,11 @@ func BrokerFactory(cnf *config.Config) (brokeriface.Broker, error) { } if strings.HasPrefix(cnf.Broker, "redis+dlq://") { - parts := strings.Split(cnf.Broker, "redis+dlq://") - if len(parts) != 2 { - return nil, fmt.Errorf( - "Redis DLQ broker connection string should be in format redis+dlq://host:port, instead got %s", - cnf.Broker, - ) + redisAddrs, redisPassword, redisDB, err := ParseRedisDlqURL(cnf.Broker) + if err != nil { + return nil, err } - brokers := strings.Split(parts[1], ",") - return redisbroker.NewGR_DLQ(cnf, brokers, 0), nil + return redisbroker.NewGR_DLQ(cnf, redisAddrs, redisPassword, redisDB), nil } if strings.HasPrefix(cnf.Broker, "redis+socket://") { @@ -262,6 +258,76 @@ func ParseRedisSocketURL(url string) (path, password string, db int, err error) return } +// ParseRedisDURL extracts Redis connection options from a URL with the +// redis+dlq:// scheme. This scheme is custom, and requires support from +// the application side as well for DLQ features such as removing messages +// which have surpasses visibilityTImeout. The broker merely sets message +// metadata in Redis when extracting tasks for processing. +func ParseRedisDlqURL(url string) (addrs []string, password string, db int, err error) { + parts := strings.Split(url, "redis+dlq://") + if parts[0] != "" { + err = errors.New("No redis scheme found") + return + } + if len(parts) != 2 { + err = fmt.Errorf( + "Redis DLQ broker connection string should be in format redis+dlq://host:port, instead got %s", + url) + return + } + + // redis+dlq://127.0.0.1:5672", + + addrs = strings.Split(parts[1], ",") + dbs := make(map[int]struct{}) + passwords := make(map[string]struct{}) + for i := range addrs { + // removing possible password from address + parts := strings.SplitN(addrs[i], "@", 2) + if len(parts) == 1 { + passwords[""] = struct{}{} + } else { + passwords[parts[0]] = struct{}{} + addrs[i] = parts[1] + } + + // go-redis URI can't handle db at the end of URI, so we're extracting that here + // note, that we can pass multiple addresses to connect in cluster/sentinel mode, but we can only pass one db + parts = strings.SplitN(addrs[i], "/", 2) + addrs[i] = parts[0] + if len(parts) == 1 { + dbs[0] = struct{}{} + } else { + db, err = strconv.Atoi(parts[1]) + if err != nil { + err = fmt.Errorf( "could not extract db from redis+dlq URI: %s", err.Error()) + return + } + dbs[db] = struct{}{} + } + } + + if len(passwords) > 1 { + err = fmt.Errorf("multiple passwords passed in redis+dlq URI. Expected one, got %d", len(passwords)) + return + } + + if len(dbs) > 1 { + err = fmt.Errorf("multiple dbs passed in redis+dlq URI. Expected one, got %d", len(dbs)) + return + } + + for k := range dbs { + db = k + } + + for pass := range passwords { + password = pass + } + + return +} + // ParseGCPPubSubURL Parse GCP Pub/Sub URL // url: gcppubsub://YOUR_GCP_PROJECT_ID/YOUR_PUBSUB_SUBSCRIPTION_NAME func ParseGCPPubSubURL(url string) (string, string, error) { diff --git a/v1/factories_test.go b/v1/factories_test.go index a0ea2b06d..201b9c39b 100644 --- a/v1/factories_test.go +++ b/v1/factories_test.go @@ -17,9 +17,9 @@ import ( redisbroker "github.com/RichardKnop/machinery/v1/brokers/redis" sqsbroker "github.com/RichardKnop/machinery/v1/brokers/sqs" - mongobackend "github.com/RichardKnop/machinery/v1/backends/mongo" amqpbackend "github.com/RichardKnop/machinery/v1/backends/amqp" memcachebackend "github.com/RichardKnop/machinery/v1/backends/memcache" + mongobackend "github.com/RichardKnop/machinery/v1/backends/mongo" redisbackend "github.com/RichardKnop/machinery/v1/backends/redis" ) @@ -187,6 +187,29 @@ func TestBrokerFactory(t *testing.T) { ) } + // using redis-dlq + cnf = config.Config{ + Broker: "redis+dlq://localhost:6379/1", + Redis: &config.RedisConfig{}, + DefaultQueue: "machinery_tasks", + } + + actual, err = machinery.BrokerFactory(&cnf) + if assert.NoError(t, err) { + _, isRedisBroker := actual.(*redisbroker.BrokerGR_DLQ) + assert.True( + t, + isRedisBroker, + "Broker should be instance of *brokers.RedisDLQBroker", + ) + expected := redisbroker.NewGR_DLQ(&cnf, []string{"localhost:6379"}, "", 1) + assert.True( + t, + brokerEqual(actual, expected), + fmt.Sprintf("conn = %+v, want %+v", actual, expected), + ) + } + // 3) AWS SQS cnf = config.Config{ Broker: "https://sqs.us-east-2.amazonaws.com/123456789012", @@ -227,6 +250,7 @@ func TestBrokerFactory(t *testing.T) { func brokerEqual(x, y brokeriface.Broker) bool { // unset Broker.stopChan and Broker.retryStopChan to nil before using // reflect.DeepEqual() as the objects will have a different address + // redisGR brokers also have objects with different addresses. Set them to nil values if they exist. rx := reflect.ValueOf(x).Elem() rxf := rx.FieldByName("stopChan") rxf = reflect.NewAt(rxf.Type(), unsafe.Pointer(rxf.UnsafeAddr())).Elem() @@ -234,6 +258,10 @@ func brokerEqual(x, y brokeriface.Broker) bool { rxf = rx.FieldByName("retryStopChan") rxf = reflect.NewAt(rxf.Type(), unsafe.Pointer(rxf.UnsafeAddr())).Elem() rxf.Set(reflect.Zero(rxf.Type())) + if rxf := rx.FieldByName("rclient"); rxf.IsValid() { + rxf = reflect.NewAt(rxf.Type(), unsafe.Pointer(rxf.UnsafeAddr())).Elem() + rxf.Set(reflect.Zero(rxf.Type())) + } ry := reflect.ValueOf(y).Elem() ryf := ry.FieldByName("stopChan") @@ -242,6 +270,10 @@ func brokerEqual(x, y brokeriface.Broker) bool { ryf = ry.FieldByName("retryStopChan") ryf = reflect.NewAt(ryf.Type(), unsafe.Pointer(ryf.UnsafeAddr())).Elem() ryf.Set(reflect.Zero(ryf.Type())) + if ryf := ry.FieldByName("rclient"); ryf.IsValid() { + ryf = reflect.NewAt(ryf.Type(), unsafe.Pointer(ryf.UnsafeAddr())).Elem() + ryf.Set(reflect.Zero(ryf.Type())) + } return reflect.DeepEqual(x, y) } @@ -470,3 +502,86 @@ func TestParseRedisSocketURL(t *testing.T) { assert.Equal(t, 2, db) } } + +func TestParseRedisDlqURL(t *testing.T) { + t.Parallel() + + var ( + hosts []string + pwd, url string + db int + err error + ) + + url = "non_redisdlq://localhost:6379" + _, _, _, err = machinery.ParseRedisDlqURL(url) + assert.Error(t, err, "No redis scheme found") + + url = "redis+dlq:/" + _, _, _, err = machinery.ParseRedisDlqURL(url) + assert.Error(t, err, "No redis scheme found") + + url = "redis+dlq://localhost:6379" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + if assert.NoError(t, err) { + assert.Equal(t, "localhost:6379", hosts[0]) + assert.Equal(t, "", pwd) + assert.Equal(t, 0, db) + } + + url = "redis+dlq://pwd@localhost:6379" + hosts, pwd, db, _ = machinery.ParseRedisDlqURL(url) + if assert.NoError(t, err) { + assert.Equal(t, "localhost:6379", hosts[0]) + assert.Equal(t, "pwd", pwd) + assert.Equal(t, 0, db) + } + + url = "redis+dlq://pwd@localhost:6379/2" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + if assert.NoError(t, err) { + assert.Equal(t, "localhost:6379", hosts[0]) + assert.Equal(t, "pwd", pwd) + assert.Equal(t, 2, db) + } + + url = "redis+dlq://localhost:6379,localhost:6380" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + if assert.NoError(t, err) { + assert.Equal(t, "localhost:6379", hosts[0]) + assert.Equal(t, "localhost:6380", hosts[1]) + assert.Equal(t, "", pwd) + assert.Equal(t, 0, db) + } + + url = "redis+dlq://pwd@localhost:6379,pwd@localhost:6380" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + if assert.NoError(t, err) { + assert.Equal(t, "localhost:6379", hosts[0]) + assert.Equal(t, "localhost:6380", hosts[1]) + assert.Equal(t, "pwd", pwd) + assert.Equal(t, 0, db) + } + + url = "redis+dlq://localhost:6379/1,localhost:6380/1" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + if assert.NoError(t, err) { + assert.Equal(t, "localhost:6379", hosts[0]) + assert.Equal(t, "localhost:6380", hosts[1]) + assert.Equal(t, "", pwd) + assert.Equal(t, 1, db) + } + + // passing no password is the same as passing an empty string password. + url = "redis+dlq://pwd@localhost:6379,localhost:6380" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + assert.Error(t, err, "multiple passwords passed in redis+dlq URI. Expected one, got 2") + + url = "redis+dlq://pwd@localhost:6379,pwd1@localhost:6380" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + assert.Error(t, err, "multiple passwords passed in redis+dlq URI. Expected one, got 2") + + url = "redis+dlq://localhost:6379,localhost:6380/3" + hosts, pwd, db, err = machinery.ParseRedisDlqURL(url) + assert.Error(t, err, "multiple dbs passed in redis+dlq URI. Expected one, got 2") +} From 13565be4aec57a34652834f55148d242087c655e Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Mon, 1 Mar 2021 22:12:08 +0500 Subject: [PATCH 07/22] feat: add more configs for use with redis clients additional configs are not usable with the default redis broker, so they've only be added for our redis-dlq broker --- v1/brokers/redis/goredis-dlq.go | 9 +++++++++ v1/config/config.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index a07a6740a..90b47bb41 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -50,6 +50,15 @@ func NewGR_DLQ(cnf *config.Config, addrs []string, password string, db int) ifac Addrs: addrs, DB: db, Password: password, + ReadTimeout: time.Duration(cnf.Redis.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(cnf.Redis.WriteTimeout) * time.Second, + DialTimeout: time.Duration(cnf.Redis.ConnectTimeout) * time.Second, + IdleTimeout: time.Duration(cnf.Redis.IdleTimeout) * time.Second, + MinIdleConns: cnf.Redis.MinIdleConns, + MinRetryBackoff: time.Duration(cnf.Redis.MinRetryBackoff) * time.Millisecond, + MaxRetryBackoff: time.Duration(cnf.Redis.MaxRetryBackoff) * time.Millisecond, + MaxRetries: cnf.Redis.MaxRetries, + PoolSize: cnf.Redis.PoolSize, } if cnf.Redis != nil { // if we're specifying MasterName here, then we'll always connect to db 0, since provided db is ignored in cluster mode diff --git a/v1/config/config.go b/v1/config/config.go index c0147d3e1..5e27abd61 100644 --- a/v1/config/config.go +++ b/v1/config/config.go @@ -137,6 +137,27 @@ type RedisConfig struct { // Default: 15 ConnectTimeout int `yaml:"connect_timeout" envconfig:"REDIS_CONNECT_TIMEOUT"` + // Minimum number of idle connections which is useful when establishing + // new connection is slow. + // Used in redis-dlq broker + MinIdleConns int `yaml:"min_idle_conns" envconfig:"REDIS_MIN_IDLE_CONNS"` + + // Minimum backoff between each retry. + // Used in redis-dlq broker + // Default: 8 * time.Millisecond + MinRetryBackoff int `yaml:"min_retry_backoff" envconfig:"REDIS_MIN_RETRY_BACKOFF"` + + // Maximum backoff between each retry. + // Used in redis-dlq broker + // Default: 512 * time.Millisecond + MaxRetryBackoff int `yaml:"max_retry_backoff" envconfig:"REDIS_MAX_RETRY_BACKOFF"` + + // Default: 0 (no retries) + MaxRetries int `yaml:"max_retries" envconfig:"REDIS_MAX_RETRIES"` + + // Default: 0 (pool size defaults to 10 * NumCpus) + PoolSize int `yaml:"pool_size" envconfig:"REDIS_POOL_SIZE"` + // VisibilityTimeout used in redis-dlq broker // default to nil to use the overall visibility timeout for redis messages VisibilityTimeout *int64 `yaml:"visibility_timeout" envconfig:"REDIS_VISIBILITY_TIMEOUT"` From eec0bb03fb857c0ecb8926902ce236391db9c63b Mon Sep 17 00:00:00 2001 From: Bharat <13206972+bharat-p@users.noreply.github.com> Date: Sat, 5 Jun 2021 07:31:58 -0700 Subject: [PATCH 08/22] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..472175841 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '45 2 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From dc4b81309e7d9900fde5001130974d9d8e690df4 Mon Sep 17 00:00:00 2001 From: Bharat Patel <13206972+bharat-p@users.noreply.github.com> Date: Sat, 5 Jun 2021 07:18:20 -0700 Subject: [PATCH 09/22] Fixes #7 Add tlsConfig when creating redis-dlq broker --- v1/brokers/redis/goredis-dlq.go | 1 + v1/brokers/redis/goredis.go | 1 + 2 files changed, 2 insertions(+) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index 90b47bb41..e8a2d3f83 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -59,6 +59,7 @@ func NewGR_DLQ(cnf *config.Config, addrs []string, password string, db int) ifac MaxRetryBackoff: time.Duration(cnf.Redis.MaxRetryBackoff) * time.Millisecond, MaxRetries: cnf.Redis.MaxRetries, PoolSize: cnf.Redis.PoolSize, + TLSConfig: cnf.TLSConfig, } if cnf.Redis != nil { // if we're specifying MasterName here, then we'll always connect to db 0, since provided db is ignored in cluster mode diff --git a/v1/brokers/redis/goredis.go b/v1/brokers/redis/goredis.go index 721ce2335..685ecc1ae 100644 --- a/v1/brokers/redis/goredis.go +++ b/v1/brokers/redis/goredis.go @@ -49,6 +49,7 @@ func NewGR(cnf *config.Config, addrs []string, db int) iface.Broker { Addrs: addrs, DB: db, Password: password, + TLSConfig: cnf.TLSConfig, } if cnf.Redis != nil { ropt.MasterName = cnf.Redis.MasterName From 4de110e353a3216eb4fb4cb13d51431ccf86f646 Mon Sep 17 00:00:00 2001 From: Uzair Khan Date: Wed, 30 Jun 2021 14:29:48 +0500 Subject: [PATCH 10/22] print old and new UUID when processing redis-dlq task --- v1/brokers/redis/goredis-dlq.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index e8a2d3f83..a0f5ca634 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -299,6 +299,7 @@ func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProce return errs.NewErrCouldNotUnmarshaTaskSignature(delivery, err) } // propagating hash UUID for possible application usage, for example, refreshing visibility + oldUuid := signature.UUID gHashByte := sha256.Sum256(delivery) gHash := fmt.Sprintf(taskPrefix, base58.Encode(gHashByte[:sha256.Size])) signature.UUID = gHash @@ -313,6 +314,7 @@ func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProce } log.DEBUG.Printf("Received new message: %+v", signature) + log.INFO.Printf("Processing task. Old UUID: %s New UUID: %s", oldUuid, signature.UUID) if err := taskProcessor.Process(signature); err != nil { // stop task deletion in case we want to send messages to dlq in redis From ced9c302070af8aed9e7e75e8a83d0aca56fc42d Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Mon, 17 Oct 2022 18:58:45 +0500 Subject: [PATCH 11/22] Add aws-sdk-v2 support Add sqs v2 broker and configuration --- go.mod | 5 +- go.sum | 60 +++-- v1/brokers/sqs/sqs.go | 8 +- v1/brokers/sqsv2/sqs.go | 404 ++++++++++++++++++++++++++++ v1/brokers/sqsv2/sqs_export_test.go | 190 +++++++++++++ v1/brokers/sqsv2/sqs_test.go | 369 +++++++++++++++++++++++++ v1/config/config.go | 8 +- v1/factories.go | 15 +- 8 files changed, 1023 insertions(+), 36 deletions(-) create mode 100644 v1/brokers/sqsv2/sqs.go create mode 100644 v1/brokers/sqsv2/sqs_export_test.go create mode 100644 v1/brokers/sqsv2/sqs_test.go diff --git a/go.mod b/go.mod index d2536db6b..70e658ffb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,9 @@ require ( github.com/RichardKnop/logging v0.0.0-20190827224416-1a693bdd4fae github.com/RichardKnop/redsync v1.2.0 github.com/aws/aws-sdk-go v1.25.8 + github.com/aws/aws-sdk-go-v2 v1.16.16 + github.com/aws/aws-sdk-go-v2/config v1.17.8 + github.com/aws/aws-sdk-go-v2/service/sqs v1.19.10 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/btcsuite/btcutil v1.0.2 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect @@ -40,7 +43,7 @@ require ( google.golang.org/appengine v1.6.5 // indirect google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect google.golang.org/grpc v1.24.0 // indirect - gopkg.in/yaml.v2 v2.2.4 + gopkg.in/yaml.v2 v2.2.8 ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999 diff --git a/go.sum b/go.sum index cceda8db7..182ebd79b 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,32 @@ github.com/RichardKnop/redsync v1.2.0/go.mod h1:9b8nBGAX3bE2uCfJGSnsDvF23mKyHTZz github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs= github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= +github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE= +github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.21 h1:4tjlyCD0hRGNQivh5dN8hbP30qQhMLBE/FgQR1vHHWM= +github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= +github.com/aws/aws-sdk-go-v2/service/sqs v1.19.10 h1:Y4civ9pg5cbQkSf/YGMfFZaIPAAAK61JV+NIzO8Ri4k= +github.com/aws/aws-sdk-go-v2/service/sqs v1.19.10/go.mod h1:65Z/rmGw/6usiOFI0Tk4ddNUmPbjjPER1WLZwnFqxFM= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 h1:pwvCchFUEnlceKIgPUouBJwK81aCkQ8UDMORfeFtW10= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 h1:OwhhKc1P9ElfWbMKPIbMMZBV6hzJlL2JKD76wNNVzgQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc6lu6OQ97U7hmdesg= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= +github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= +github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= @@ -38,7 +64,6 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -55,14 +80,12 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc h1:55rEp52jU6bkyslZ1+C/7NGfpQsEc6pxGLAGDOctqbw= github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -73,10 +96,10 @@ github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNu github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -92,10 +115,12 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -129,7 +154,6 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -153,14 +177,11 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979 h1:Agxu5KLo8o7Bb634SVDnhIfpTvxmzUwhbYAzBvXt6h4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20190912063710-ac5d2bfcbfe0/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs= @@ -172,7 +193,6 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac h1:8R1esu+8QioDxo4E4mX6bFztO+dMTM49DNAaWfO5OeY= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -184,7 +204,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd h1:HuTn7WObtcDo9uEEU7rEqL0jYthdXAmZ6PP+meazmaU= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -198,17 +217,14 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -219,9 +235,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -240,7 +254,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff h1:On1qIo75ByTwFJ4/W2bIqHcwJ9XAqtSWUs8GwRrIhtc= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190917162342-3b4f30a44f3b/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -253,9 +266,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.11.0 h1:n/qM3q0/rV2F0pox7o0CvNhlPvZAo7pLbef122cbLJ0= google.golang.org/api v0.11.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -276,7 +287,6 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -285,12 +295,10 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/v1/brokers/sqs/sqs.go b/v1/brokers/sqs/sqs.go index eccecd974..56084ba06 100644 --- a/v1/brokers/sqs/sqs.go +++ b/v1/brokers/sqs/sqs.go @@ -48,9 +48,9 @@ type ReceivedMessages struct { // New creates new Broker instance func New(cnf *config.Config) iface.Broker { b := &Broker{Broker: common.NewBroker(cnf)} - if cnf.SQS != nil && cnf.SQS.Client != nil { + if cnf.SQS != nil && cnf.SQS.ClientV1 != nil { // Use provided *SQS client - b.service = cnf.SQS.Client + b.service = cnf.SQS.ClientV1 } else { // Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials. // See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html @@ -91,7 +91,7 @@ func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcess qURL := b.getQueueURL(taskProcessor) output, err := b.receiveMessage(qURL) if err == nil && len(output.Messages) > 0 { - deliveries <- &ReceivedMessages{Delivery: output, queue:qURL} + deliveries <- &ReceivedMessages{Delivery: output, queue: qURL} } else { //return back to pool right away @@ -377,6 +377,6 @@ func (b *Broker) queueToURL(queue string) *string { func (b *Broker) urlToQueue(queue string) *string { parts := strings.Split(queue, "/") - queueName := parts[len(parts) - 1] + queueName := parts[len(parts)-1] return &queueName } diff --git a/v1/brokers/sqsv2/sqs.go b/v1/brokers/sqsv2/sqs.go new file mode 100644 index 000000000..cd096a60d --- /dev/null +++ b/v1/brokers/sqsv2/sqs.go @@ -0,0 +1,404 @@ +package sqsv2 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/RichardKnop/machinery/v1/brokers/errs" + "github.com/RichardKnop/machinery/v1/brokers/iface" + "github.com/RichardKnop/machinery/v1/common" + "github.com/RichardKnop/machinery/v1/config" + "github.com/RichardKnop/machinery/v1/log" + "github.com/RichardKnop/machinery/v1/tasks" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" +) + +const ( + maxAWSSQSDelay = time.Minute * 15 // Max supported SQS delay is 15 min: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html +) + +type SQSAPIV2 interface { + AddPermission(ctx context.Context, params *awssqs.AddPermissionInput, optFns ...func(*awssqs.Options)) (*awssqs.AddPermissionOutput, error) + ChangeMessageVisibility(ctx context.Context, params *awssqs.ChangeMessageVisibilityInput, optFns ...func(*awssqs.Options)) (*awssqs.ChangeMessageVisibilityOutput, error) + ChangeMessageVisibilityBatch(ctx context.Context, params *awssqs.ChangeMessageVisibilityBatchInput, optFns ...func(*awssqs.Options)) (*awssqs.ChangeMessageVisibilityBatchOutput, error) + CreateQueue(ctx context.Context, params *awssqs.CreateQueueInput, optFns ...func(*awssqs.Options)) (*awssqs.CreateQueueOutput, error) + DeleteMessage(ctx context.Context, params *awssqs.DeleteMessageInput, optFns ...func(*awssqs.Options)) (*awssqs.DeleteMessageOutput, error) + DeleteMessageBatch(ctx context.Context, params *awssqs.DeleteMessageBatchInput, optFns ...func(*awssqs.Options)) (*awssqs.DeleteMessageBatchOutput, error) + DeleteQueue(ctx context.Context, params *awssqs.DeleteQueueInput, optFns ...func(*awssqs.Options)) (*awssqs.DeleteQueueOutput, error) + GetQueueAttributes(ctx context.Context, params *awssqs.GetQueueAttributesInput, optFns ...func(*awssqs.Options)) (*awssqs.GetQueueAttributesOutput, error) + GetQueueUrl(ctx context.Context, params *awssqs.GetQueueUrlInput, optFns ...func(*awssqs.Options)) (*awssqs.GetQueueUrlOutput, error) + ListDeadLetterSourceQueues(ctx context.Context, params *awssqs.ListDeadLetterSourceQueuesInput, optFns ...func(*awssqs.Options)) (*awssqs.ListDeadLetterSourceQueuesOutput, error) + ListQueueTags(ctx context.Context, params *awssqs.ListQueueTagsInput, optFns ...func(*awssqs.Options)) (*awssqs.ListQueueTagsOutput, error) + ListQueues(ctx context.Context, params *awssqs.ListQueuesInput, optFns ...func(*awssqs.Options)) (*awssqs.ListQueuesOutput, error) + PurgeQueue(ctx context.Context, params *awssqs.PurgeQueueInput, optFns ...func(*awssqs.Options)) (*awssqs.PurgeQueueOutput, error) + ReceiveMessage(ctx context.Context, params *awssqs.ReceiveMessageInput, optFns ...func(*awssqs.Options)) (*awssqs.ReceiveMessageOutput, error) + RemovePermission(ctx context.Context, params *awssqs.RemovePermissionInput, optFns ...func(*awssqs.Options)) (*awssqs.RemovePermissionOutput, error) + SendMessage(ctx context.Context, params *awssqs.SendMessageInput, optFns ...func(*awssqs.Options)) (*awssqs.SendMessageOutput, error) + SendMessageBatch(ctx context.Context, params *awssqs.SendMessageBatchInput, optFns ...func(*awssqs.Options)) (*awssqs.SendMessageBatchOutput, error) + SetQueueAttributes(ctx context.Context, params *awssqs.SetQueueAttributesInput, optFns ...func(*awssqs.Options)) (*awssqs.SetQueueAttributesOutput, error) + TagQueue(ctx context.Context, params *awssqs.TagQueueInput, optFns ...func(*awssqs.Options)) (*awssqs.TagQueueOutput, error) + UntagQueue(ctx context.Context, params *awssqs.UntagQueueInput, optFns ...func(*awssqs.Options)) (*awssqs.UntagQueueOutput, error) +} + +// Broker represents a AWS SQS broker +// There are examples on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sqs-example-create-queue.html +type Broker struct { + common.Broker + processingWG sync.WaitGroup // use wait group to make sure task processing completes on interrupt signal + receivingWG sync.WaitGroup + stopReceivingChan chan int + config aws.Config + service SQSAPIV2 +} + +// ReceivedMessages contains the queue name that the received message was fetched from so that we can delete the message later +// Since machinery can fetch messages from multiple source queues now, we need to either embed the source queue name inside the +// message signature (which can be prone to errors), or use this. +type ReceivedMessages struct { + Delivery *awssqs.ReceiveMessageOutput + queue *string +} + +// New creates new Broker instance +func New(cnf *config.Config) iface.Broker { + b := &Broker{Broker: common.NewBroker(cnf)} + if cnf.SQS != nil && cnf.SQS.ClientV2 != nil { + // Use provided *SQS client + b.service = cnf.SQS.ClientV2 + } else { + // Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials. + // See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html + // Also, env AWS_REGION is also required + cfg, _ := awscfg.LoadDefaultConfig(context.TODO()) + b.config = cfg + b.service = awssqs.NewFromConfig(cfg) + } + + return b +} + +// StartConsuming enters a loop and waits for incoming messages +func (b *Broker) StartConsuming(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) (bool, error) { + b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) + deliveries := make(chan *ReceivedMessages, concurrency) + pool := make(chan struct{}, concurrency) + + // initialize worker pool with maxWorkers workers + for i := 0; i < concurrency; i++ { + pool <- struct{}{} + } + b.stopReceivingChan = make(chan int) + b.receivingWG.Add(1) + + go func() { + defer b.receivingWG.Done() + + log.INFO.Printf("[*] Waiting for messages on queue. To exit press CTRL+C\n") + for { + select { + // A way to stop this goroutine from b.StopConsuming + case <-b.stopReceivingChan: + close(deliveries) + return + case <-pool: + qURL := b.getQueueURL(taskProcessor) + output, err := b.receiveMessage(qURL) + if err == nil && len(output.Messages) > 0 { + deliveries <- &ReceivedMessages{Delivery: output, queue: qURL} + + } else { + //return back to pool right away + pool <- struct{}{} + if err != nil { + log.ERROR.Printf("Queue consume error: %s", err) + } + + } + } + + } + }() + + if err := b.consume(deliveries, concurrency, taskProcessor, pool); err != nil { + return b.GetRetry(), err + } + + return b.GetRetry(), nil +} + +// StopConsuming quits the loop +func (b *Broker) StopConsuming() { + b.Broker.StopConsuming() + + b.stopReceiving() + + // Waiting for any tasks being processed to finish + b.processingWG.Wait() + + // Waiting for the receiving goroutine to have stopped + b.receivingWG.Wait() +} + +// Publish places a new message on the default queue +func (b *Broker) Publish(ctx context.Context, signature *tasks.Signature) error { + msg, err := json.Marshal(signature) + if err != nil { + return fmt.Errorf("JSON marshal error: %s", err) + } + + // Check that signature.RoutingKey is set, if not switch to DefaultQueue + b.AdjustRoutingKey(signature) + + MsgInput := &awssqs.SendMessageInput{ + MessageBody: aws.String(string(msg)), + QueueUrl: b.queueToURL(signature.RoutingKey), + } + + // if this is a fifo queue, there needs to be some additional parameters. + if strings.HasSuffix(signature.RoutingKey, ".fifo") { + // Use Machinery's signature Task UUID as SQS Message Group ID. + MsgDedupID := signature.UUID + MsgInput.MessageDeduplicationId = aws.String(MsgDedupID) + + // Do not Use Machinery's signature Group UUID as SQS Message Group ID, instead use BrokerMessageGroupId + MsgGroupID := signature.BrokerMessageGroupId + if MsgGroupID == "" { + return fmt.Errorf("please specify BrokerMessageGroupId attribute for task Signature when submitting a task to FIFO queue") + } + MsgInput.MessageGroupId = aws.String(MsgGroupID) + } + + // Check the ETA signature field, if it is set and it is in the future, + // and is not a fifo queue, set a delay in seconds for the task. + if signature.ETA != nil && !strings.HasSuffix(signature.RoutingKey, ".fifo") { + now := time.Now().UTC() + delay := signature.ETA.Sub(now) + if delay > 0 { + if delay > maxAWSSQSDelay { + return errors.New("Max AWS SQS delay exceeded") + } + MsgInput.DelaySeconds = int32(delay.Seconds()) + } + } + + result, err := b.service.SendMessage(ctx, MsgInput) + + if err != nil { + log.ERROR.Printf("Error when sending a message: %v", err) + return err + + } + log.INFO.Printf("Sending a message successfully, the messageId is %v", *result.MessageId) + return nil + +} + +// consume is a method which keeps consuming deliveries from a channel, until there is an error or a stop signal +func (b *Broker) consume(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { + + errorsChan := make(chan error) + + for { + whetherContinue, err := b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) + if err != nil { + return err + } + if whetherContinue == false { + return nil + } + } +} + +// consumeOne is a method consumes a delivery. If a delivery was consumed successfully, it will be deleted from AWS SQS +func (b *Broker) consumeOne(sqsReceivedMsgs *ReceivedMessages, taskProcessor iface.TaskProcessor) error { + delivery := sqsReceivedMsgs.Delivery + if len(delivery.Messages) == 0 { + log.ERROR.Printf("received an empty message, the delivery was %v", delivery) + return fmt.Errorf("received empty message, the delivery is %+v", delivery) + } + + sig := new(tasks.Signature) + decoder := json.NewDecoder(strings.NewReader(*delivery.Messages[0].Body)) + decoder.UseNumber() + if err := decoder.Decode(sig); err != nil { + log.ERROR.Printf("unmarshal error. the delivery is %v", delivery) + // if the unmarshal fails, remove the delivery from the queue + if delErr := b.deleteOne(sqsReceivedMsgs); delErr != nil { + log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, delErr) + } + return err + } + if delivery.Messages[0].ReceiptHandle != nil { + sig.SQSReceiptHandle = *delivery.Messages[0].ReceiptHandle + } + + // If the task is not registered return an error + // and leave the message in the queue + if !b.IsTaskRegistered(sig.Name) { + if sig.IgnoreWhenTaskNotRegistered { + b.deleteOne(sqsReceivedMsgs) + } + return fmt.Errorf("task %s is not registered", sig.Name) + } + + if sqsReceivedMsgs.queue != nil { + queueName := b.urlToQueue(*(sqsReceivedMsgs.queue)) + sig.RoutingKey = *queueName + } + + err := taskProcessor.Process(sig) + if err != nil { + // stop task deletion in case we want to send messages to dlq in sqs + if err == errs.ErrStopTaskDeletion { + return nil + } + return err + } + // Delete message after successfully consuming and processing the message + if err = b.deleteOne(sqsReceivedMsgs); err != nil { + log.ERROR.Printf("error when deleting the delivery. delivery is %v, Error=%s", delivery, err) + } + return err +} + +// deleteOne is a method delete a delivery from AWS SQS +func (b *Broker) deleteOne(delivery *ReceivedMessages) error { + qURL := delivery.queue + + _, err := b.service.DeleteMessage(context.TODO(), &awssqs.DeleteMessageInput{ + QueueUrl: qURL, + ReceiptHandle: delivery.Delivery.Messages[0].ReceiptHandle, + }) + + if err != nil { + return err + } + return nil +} + +// defaultQueueURL is a method returns the default queue url +func (b *Broker) defaultQueueURL() *string { + return b.queueToURL(b.GetConfig().DefaultQueue) +} + +// receiveMessage is a method receives a message from specified queue url +func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, error) { + var waitTimeSeconds int + var visibilityTimeout *int + if b.GetConfig().SQS != nil { + waitTimeSeconds = b.GetConfig().SQS.WaitTimeSeconds + visibilityTimeout = b.GetConfig().SQS.VisibilityTimeout + } else { + waitTimeSeconds = 0 + } + input := &awssqs.ReceiveMessageInput{ + AttributeNames: []types.QueueAttributeName{ + types.QueueAttributeName(types.MessageSystemAttributeNameSentTimestamp), + }, + MessageAttributeNames: []string{ + string(types.QueueAttributeNameAll), + }, + QueueUrl: qURL, + MaxNumberOfMessages: 1, + WaitTimeSeconds: int32(waitTimeSeconds), + } + if visibilityTimeout != nil { + input.VisibilityTimeout = int32(*visibilityTimeout) + } + result, err := b.service.ReceiveMessage(context.TODO(), input) + if err != nil { + return nil, err + } + return result, err +} + +// initializePool is a method which initializes concurrency pool +func (b *Broker) initializePool(pool chan struct{}, concurrency int) { + for i := 0; i < concurrency; i++ { + pool <- struct{}{} + } +} + +// consumeDeliveries is a method consuming deliveries from deliveries channel +func (b *Broker) consumeDeliveries(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { + select { + case err := <-errorsChan: + return false, err + case d := <-deliveries: + + b.processingWG.Add(1) + + // Consume the task inside a goroutine so multiple tasks + // can be processed concurrently + go func() { + + if err := b.consumeOne(d, taskProcessor); err != nil { + errorsChan <- err + } + + b.processingWG.Done() + + if concurrency > 0 { + // give worker back to pool + pool <- struct{}{} + } + }() + case <-b.GetStopChan(): + return false, nil + } + return true, nil +} + +// continueReceivingMessages is a method returns a continue signal +func (b *Broker) continueReceivingMessages(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) { + select { + // A way to stop this goroutine from b.StopConsuming + case <-b.stopReceivingChan: + return false, nil + default: + output, err := b.receiveMessage(qURL) + if err != nil { + return true, err + } + if len(output.Messages) == 0 { + return true, nil + } + go func() { deliveries <- output }() + } + return true, nil +} + +// stopReceiving is a method sending a signal to stopReceivingChan +func (b *Broker) stopReceiving() { + // Stop the receiving goroutine + b.stopReceivingChan <- 1 +} + +// getQueueURL is a method returns that returns queueURL first by checking if custom queue was set and usign it +// otherwise using default queueName from config +func (b *Broker) getQueueURL(taskProcessor iface.TaskProcessor) *string { + if customQueue := taskProcessor.CustomQueue(); customQueue != "" { + return b.queueToURL(customQueue) + } + + return b.defaultQueueURL() +} + +func (b *Broker) queueToURL(queue string) *string { + return aws.String(b.GetConfig().Broker + "/" + queue) +} + +func (b *Broker) urlToQueue(queue string) *string { + parts := strings.Split(queue, "/") + queueName := parts[len(parts)-1] + return &queueName +} diff --git a/v1/brokers/sqsv2/sqs_export_test.go b/v1/brokers/sqsv2/sqs_export_test.go new file mode 100644 index 000000000..d3d015ca0 --- /dev/null +++ b/v1/brokers/sqsv2/sqs_export_test.go @@ -0,0 +1,190 @@ +package sqsv2 + +import ( + "context" + "encoding/json" + "errors" + "os" + "sync" + + "github.com/RichardKnop/machinery/v1/brokers/iface" + "github.com/RichardKnop/machinery/v1/common" + "github.com/RichardKnop/machinery/v1/config" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" +) + +var ( + TestAWSSQSBroker *Broker + ErrAWSSQSBroker *Broker + ReceiveMessageOutput *awssqs.ReceiveMessageOutput + TestConf *config.Config +) + +type FakeSQS struct { + SQSAPIV2 +} + +func (f *FakeSQS) SendMessage(_ context.Context, _ *awssqs.SendMessageInput, _ ...func(*awssqs.Options)) (*awssqs.SendMessageOutput, error) { + output := awssqs.SendMessageOutput{ + MD5OfMessageAttributes: aws.String("d25a6aea97eb8f585bfa92d314504a92"), + MD5OfMessageBody: aws.String("bbdc5fdb8be7251f5c910905db994bab"), + MessageId: aws.String("47f8b355-5115-4b45-b33a-439016400411"), + } + return &output, nil +} + +func (f *FakeSQS) ReceiveMessage(_ context.Context, _ *awssqs.ReceiveMessageInput, _ ...func(*awssqs.Options)) (*awssqs.ReceiveMessageOutput, error) { + return ReceiveMessageOutput, nil +} + +func (f *FakeSQS) DeleteMessage(_ context.Context, _ *awssqs.DeleteMessageInput, _ ...func(*awssqs.Options)) (*awssqs.DeleteMessageOutput, error) { + return &awssqs.DeleteMessageOutput{}, nil +} + +type ErrorSQS struct { + SQSAPIV2 +} + +func (f *ErrorSQS) SendMessage(_ context.Context, _ *awssqs.SendMessageInput, _ ...func(*awssqs.Options)) (*awssqs.SendMessageOutput, error) { + err := errors.New("this is an error") + return nil, err +} + +func (f *ErrorSQS) ReceiveMessage(_ context.Context, _ *awssqs.ReceiveMessageInput, _ ...func(*awssqs.Options)) (*awssqs.ReceiveMessageOutput, error) { + err := errors.New("this is an error") + return nil, err +} + +func (f *ErrorSQS) DeleteMessage(_ context.Context, _ *awssqs.DeleteMessageInput, _ ...func(*awssqs.Options)) (*awssqs.DeleteMessageOutput, error) { + err := errors.New("this is an error") + return nil, err +} + +func init() { + redisURL := os.Getenv("REDIS_URL") + brokerURL := "https://sqs.foo.amazonaws.com.cn" + TestConf = &config.Config{ + Broker: brokerURL, + DefaultQueue: "test_queue", + ResultBackend: redisURL, + } + cfg, _ := awscfg.LoadDefaultConfig(context.TODO()) + svc := new(FakeSQS) + TestAWSSQSBroker = &Broker{ + Broker: common.NewBroker(TestConf), + config: cfg, + service: svc, + processingWG: sync.WaitGroup{}, + receivingWG: sync.WaitGroup{}, + stopReceivingChan: make(chan int), + } + + errSvc := new(ErrorSQS) + ErrAWSSQSBroker = &Broker{ + Broker: common.NewBroker(TestConf), + config: cfg, + service: errSvc, + processingWG: sync.WaitGroup{}, + receivingWG: sync.WaitGroup{}, + stopReceivingChan: make(chan int), + } + + // TODO: chang message body to signature example + messageBody, _ := json.Marshal(map[string]int{"apple": 5, "lettuce": 7}) + ReceiveMessageOutput = &awssqs.ReceiveMessageOutput{ + Messages: []types.Message{ + { + Attributes: map[string]string{ + "SentTimestamp": "1512962021537", + }, + Body: aws.String(string(messageBody)), + MD5OfBody: aws.String("bbdc5fdb8be7251f5c910905db994bab"), + MD5OfMessageAttributes: aws.String("d25a6aea97eb8f585bfa92d314504a92"), + MessageAttributes: map[string]types.MessageAttributeValue{ + "Title": { + DataType: aws.String("String"), + StringValue: aws.String("The Whistler"), + }, + "Author": { + DataType: aws.String("String"), + StringValue: aws.String("John Grisham"), + }, + "WeeksOn": { + DataType: aws.String("Number"), + StringValue: aws.String("6"), + }, + }, + MessageId: aws.String("47f8b355-5115-4b45-b33a-439016400411"), + ReceiptHandle: aws.String("AQEBGhTR/nhq+pDPAunCDgLpwQuCq0JkD2dtv7pAcPF5DA/XaoPAjHfgn/PZ5DeG3YiQdTjCUj+rvFq5b79DTq+hK6r1Niuds02l+jdIk3u2JiL01Dsd203pW1lLUNryd74QAcn462eXzv7/hVDagXTn+KtOzox3X0vmPkCSQkWXWxtc23oa5+5Q7HWDmRm743L0zza1579rQ2R2B0TrdlTMpNsdjQlDmybNu+aDq8bazD/Wew539tIvUyYADuhVyKyS1L2QQuyXll73/DixulPNmvGPRHNoB1GIo+Ex929OHFchXoKonoFJnurX4VNNl1p/Byp2IYBi6nkTRzeJUFCrFq0WMAHKLwuxciezJSlLD7g3bbU8kgEer8+jTz1DBriUlDGsARr0s7mnlsd02cb46K/j+u1oPfA69vIVc0FaRtA="), + }, + }, + } +} + +func (b *Broker) ConsumeForTest(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}) error { + return b.consume(deliveries, concurrency, taskProcessor, pool) +} + +func (b *Broker) ConsumeOneForTest(delivery *ReceivedMessages, taskProcessor iface.TaskProcessor) error { + return b.consumeOne(delivery, taskProcessor) +} + +func (b *Broker) DeleteOneForTest(delivery *ReceivedMessages) error { + return b.deleteOne(delivery) +} + +func (b *Broker) DefaultQueueURLForTest() *string { + return b.defaultQueueURL() +} + +func (b *Broker) ReceiveMessageForTest(qURL *string) (*awssqs.ReceiveMessageOutput, error) { + return b.receiveMessage(qURL) +} + +func (b *Broker) InitializePoolForTest(pool chan struct{}, concurrency int) { + b.initializePool(pool, concurrency) +} + +func (b *Broker) ConsumeDeliveriesForTest(deliveries <-chan *ReceivedMessages, concurrency int, taskProcessor iface.TaskProcessor, pool chan struct{}, errorsChan chan error) (bool, error) { + return b.consumeDeliveries(deliveries, concurrency, taskProcessor, pool, errorsChan) +} + +func (b *Broker) ContinueReceivingMessagesForTest(qURL *string, deliveries chan *awssqs.ReceiveMessageOutput) (bool, error) { + return b.continueReceivingMessages(qURL, deliveries) +} + +func (b *Broker) StopReceivingForTest() { + b.stopReceiving() +} + +func (b *Broker) GetStopReceivingChanForTest() chan int { + return b.stopReceivingChan +} + +func (b *Broker) StartConsumingForTest(consumerTag string, concurrency int, taskProcessor iface.TaskProcessor) { + b.Broker.StartConsuming(consumerTag, concurrency, taskProcessor) +} + +func (b *Broker) GetRetryFuncForTest() func(chan int) { + return b.GetRetryFunc() +} + +func (b *Broker) GetStopChanForTest() chan int { + return b.GetStopChan() +} + +func (b *Broker) GetRetryStopChanForTest() chan int { + return b.GetRetryStopChan() +} + +func (b *Broker) GetQueueURLForTest(taskProcessor iface.TaskProcessor) *string { + return b.getQueueURL(taskProcessor) +} + +func (b *Broker) GetCustomQueueURL(customQueue string) *string { + return aws.String(b.GetConfig().Broker + "/" + customQueue) +} diff --git a/v1/brokers/sqsv2/sqs_test.go b/v1/brokers/sqsv2/sqs_test.go new file mode 100644 index 000000000..05795149c --- /dev/null +++ b/v1/brokers/sqsv2/sqs_test.go @@ -0,0 +1,369 @@ +package sqsv2_test + +import ( + "context" + "errors" + "github.com/RichardKnop/machinery/v1/brokers/iface" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/RichardKnop/machinery/v1" + "github.com/RichardKnop/machinery/v1/brokers/sqsv2" + "github.com/RichardKnop/machinery/v1/config" + "github.com/RichardKnop/machinery/v1/retry" + + "github.com/aws/aws-sdk-go-v2/aws" + awssqs "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" +) + +var ( + testBroker iface.Broker + testAWSSQSBroker *sqsv2.Broker + errAWSSQSBroker *sqsv2.Broker + cnf *config.Config + receiveMessageOutput *awssqs.ReceiveMessageOutput +) + +func init() { + testAWSSQSBroker = sqsv2.TestAWSSQSBroker + errAWSSQSBroker = sqsv2.ErrAWSSQSBroker + cnf = sqsv2.TestConf + receiveMessageOutput = sqsv2.ReceiveMessageOutput + testBroker = sqsv2.New(cnf) +} + +func TestNewAWSSQSBroker(t *testing.T) { + assert.IsType(t, testAWSSQSBroker, testBroker) +} + +func TestPrivateFunc_continueReceivingMessages(t *testing.T) { + qURL := testAWSSQSBroker.DefaultQueueURLForTest() + deliveries := make(chan *awssqs.ReceiveMessageOutput) + firstStep := make(chan int) + nextStep := make(chan int) + go func() { + stopReceivingChan := testAWSSQSBroker.GetStopReceivingChanForTest() + firstStep <- 1 + stopReceivingChan <- 1 + }() + + var ( + whetherContinue bool + err error + ) + <-firstStep + // Test the case that a signal was received from stopReceivingChan + go func() { + whetherContinue, err = testAWSSQSBroker.ContinueReceivingMessagesForTest(qURL, deliveries) + nextStep <- 1 + }() + <-nextStep + assert.False(t, whetherContinue) + assert.Nil(t, err) + + // Test the default condition + whetherContinue, err = testAWSSQSBroker.ContinueReceivingMessagesForTest(qURL, deliveries) + assert.True(t, whetherContinue) + assert.Nil(t, err) + + // Test the error + whetherContinue, err = errAWSSQSBroker.ContinueReceivingMessagesForTest(qURL, deliveries) + assert.True(t, whetherContinue) + assert.NotNil(t, err) + + // Test when there is no message + outputCopy := *receiveMessageOutput + receiveMessageOutput.Messages = []types.Message{} + whetherContinue, err = testAWSSQSBroker.ContinueReceivingMessagesForTest(qURL, deliveries) + assert.True(t, whetherContinue) + assert.Nil(t, err) + // recover original value + *receiveMessageOutput = outputCopy + +} + +func TestPrivateFunc_consume(t *testing.T) { + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + pool := make(chan struct{}, 0) + wk := server1.NewWorker("sms_worker", 0) + deliveries := make(chan *sqsv2.ReceivedMessages) + outputCopy := *receiveMessageOutput + outputCopy.Messages = []types.Message{} + receivedMsg := sqsv2.ReceivedMessages{Delivery: &outputCopy} + go func() { deliveries <- &receivedMsg }() + + // an infinite loop will be executed only when there is no error + err = testAWSSQSBroker.ConsumeForTest(deliveries, 0, wk, pool) + assert.NotNil(t, err) + +} + +func TestPrivateFunc_consumeOne(t *testing.T) { + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + wk := server1.NewWorker("sms_worker", 0) + outputCopy := *receiveMessageOutput + outputCopy.Messages = []types.Message{} + receivedMsg := sqsv2.ReceivedMessages{Delivery: &outputCopy} + err = testAWSSQSBroker.ConsumeOneForTest(&receivedMsg, wk) + assert.NotNil(t, err) + + err = testAWSSQSBroker.ConsumeOneForTest(&receivedMsg, wk) + assert.NotNil(t, err) + + outputCopy.Messages = []types.Message{ + { + Body: aws.String("foo message"), + }, + } + receivedMsg = sqsv2.ReceivedMessages{Delivery: &outputCopy} + err = testAWSSQSBroker.ConsumeOneForTest(&receivedMsg, wk) + assert.NotNil(t, err) +} + +func TestPrivateFunc_initializePool(t *testing.T) { + concurrency := 9 + pool := make(chan struct{}, concurrency) + testAWSSQSBroker.InitializePoolForTest(pool, concurrency) + assert.Len(t, pool, concurrency) +} + +func TestPrivateFunc_startConsuming(t *testing.T) { + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + + wk := server1.NewWorker("sms_worker", 0) + retryFunc := testAWSSQSBroker.GetRetryFuncForTest() + stopChan := testAWSSQSBroker.GetStopChanForTest() + retryStopChan := testAWSSQSBroker.GetRetryStopChanForTest() + assert.Nil(t, retryFunc) + testAWSSQSBroker.StartConsumingForTest("fooTag", 1, wk) + assert.IsType(t, retryFunc, retry.Closure()) + assert.Equal(t, len(stopChan), 0) + assert.Equal(t, len(retryStopChan), 0) +} + +func TestPrivateFuncDefaultQueueURL(t *testing.T) { + qURL := testAWSSQSBroker.DefaultQueueURLForTest() + + assert.EqualValues(t, *qURL, "https://sqs.foo.amazonaws.com.cn/test_queue") +} + +func TestPrivateFunc_stopReceiving(t *testing.T) { + go testAWSSQSBroker.StopReceivingForTest() + stopReceivingChan := testAWSSQSBroker.GetStopReceivingChanForTest() + assert.NotNil(t, <-stopReceivingChan) +} + +func TestPrivateFunc_receiveMessage(t *testing.T) { + qURL := testAWSSQSBroker.DefaultQueueURLForTest() + output, err := testAWSSQSBroker.ReceiveMessageForTest(qURL) + assert.Nil(t, err) + assert.Equal(t, receiveMessageOutput, output) +} + +func TestPrivateFunc_consumeDeliveries(t *testing.T) { + concurrency := 0 + pool := make(chan struct{}, concurrency) + errorsChan := make(chan error) + deliveries := make(chan *sqsv2.ReceivedMessages) + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + wk := server1.NewWorker("sms_worker", 0) + outputCopy := *receiveMessageOutput + outputCopy.Messages = []types.Message{} + receivedMsg := sqsv2.ReceivedMessages{Delivery: &outputCopy} + go func() { deliveries <- &receivedMsg }() + whetherContinue, err := testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) + assert.True(t, whetherContinue) + assert.Nil(t, err) + + go func() { errorsChan <- errors.New("foo error") }() + whetherContinue, err = testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) + assert.False(t, whetherContinue) + assert.NotNil(t, err) + + go func() { testAWSSQSBroker.GetStopChanForTest() <- 1 }() + whetherContinue, err = testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) + assert.False(t, whetherContinue) + assert.Nil(t, err) + + outputCopy = *receiveMessageOutput + outputCopy.Messages = []types.Message{} + receivedMsg = sqsv2.ReceivedMessages{Delivery: &outputCopy} + go func() { deliveries <- &receivedMsg }() + whetherContinue, err = testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) + e := <-errorsChan + assert.True(t, whetherContinue) + assert.NotNil(t, e) + assert.Nil(t, err) + + // using a wait group and a channel to fix the racing problem + outputCopy = *receiveMessageOutput + outputCopy.Messages = []types.Message{} + receivedMsg = sqsv2.ReceivedMessages{Delivery: &outputCopy} + var wg sync.WaitGroup + wg.Add(1) + nextStep := make(chan bool, 1) + go func() { + defer wg.Done() + // nextStep <- true runs after defer wg.Done(), to make sure the next go routine runs after this go routine + nextStep <- true + deliveries <- &receivedMsg + }() + if <-nextStep { + // <-pool will block the routine in the following steps, so pool <- struct{}{} will be executed for sure + go func() { wg.Wait(); pool <- struct{}{} }() + } + whetherContinue, err = testAWSSQSBroker.ConsumeDeliveriesForTest(deliveries, concurrency, wk, pool, errorsChan) + // the pool shouldn't be consumed + p := <-pool + assert.True(t, whetherContinue) + assert.NotNil(t, p) + assert.Nil(t, err) +} + +func TestPrivateFunc_deleteOne(t *testing.T) { + outputCopy := *receiveMessageOutput + outputCopy.Messages = []types.Message{ + { + Body: aws.String("foo message"), + ReceiptHandle: aws.String("receipthandle"), + }, + } + receivedMsg := sqsv2.ReceivedMessages{Delivery: &outputCopy} + err := testAWSSQSBroker.DeleteOneForTest(&receivedMsg) + assert.Nil(t, err) + + err = errAWSSQSBroker.DeleteOneForTest(&receivedMsg) + assert.NotNil(t, err) +} + +func Test_CustomQueueName(t *testing.T) { + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + + wk := server1.NewWorker("test-worker", 0) + qURL := testAWSSQSBroker.GetQueueURLForTest(wk) + assert.Equal(t, qURL, testAWSSQSBroker.DefaultQueueURLForTest(), "") + + wk2 := server1.NewCustomQueueWorker("test-worker", 0, "my-custom-queue") + qURL2 := testAWSSQSBroker.GetQueueURLForTest(wk2) + assert.Equal(t, qURL2, testAWSSQSBroker.GetCustomQueueURL("my-custom-queue"), "") +} + +func TestPrivateFunc_consumeWithConcurrency(t *testing.T) { + + msg := `{ + "UUID": "uuid-dummy-task", + "Name": "test-task", + "RoutingKey": "dummy-routing" + } + ` + + testResp := "47f8b355-5115-4b45-b33a-439016400411" + output := make(chan string) // The output channel + + cnf.ResultBackend = "eager" + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + err = server1.RegisterTask("test-task", func(ctx context.Context) error { + output <- testResp + + return nil + }) + testAWSSQSBroker.SetRegisteredTaskNames([]string{"test-task"}) + assert.NoError(t, err) + pool := make(chan struct{}, 1) + pool <- struct{}{} + wk := server1.NewWorker("sms_worker", 1) + deliveries := make(chan *sqsv2.ReceivedMessages) + outputCopy := *receiveMessageOutput + outputCopy.Messages = []types.Message{ + { + MessageId: aws.String("test-sqs-msg1"), + Body: aws.String(msg), + }, + } + receivedMsg := sqsv2.ReceivedMessages{Delivery: &outputCopy} + go func() { + deliveries <- &receivedMsg + + }() + + go func() { + err = testAWSSQSBroker.ConsumeForTest(deliveries, 1, wk, pool) + }() + + select { + case resp := <-output: + assert.Equal(t, testResp, resp) + + case <-time.After(10 * time.Second): + // call timed out + t.Fatal("task not processed in 10 seconds") + } +} + +type roundRobinQueues struct { + queues []string + currentIndex int +} + +func NewRoundRobinQueues(queues []string) *roundRobinQueues { + return &roundRobinQueues{ + queues: queues, + currentIndex: -1, + } +} + +func (r *roundRobinQueues) Peek() string { + return r.queues[r.currentIndex] +} + +func (r *roundRobinQueues) Next() string { + r.currentIndex += 1 + if r.currentIndex >= len(r.queues) { + r.currentIndex = 0 + } + + q := r.queues[r.currentIndex] + return q +} + +func TestPrivateFunc_consumeWithRoundRobinQueues(t *testing.T) { + server1, err := machinery.NewServer(cnf) + if err != nil { + t.Fatal(err) + } + + w := server1.NewWorker("test-worker", 0) + + // Assigning a getQueueHandler to `Next` method of roundRobinQueues + rr := NewRoundRobinQueues([]string{"custom-queue-0", "custom-queue-1", "custom-queue-2", "custom-queue-3"}) + w.SetGetQueueHandler(rr.Next) + + for i := 0; i < 5; i++ { + // the queue url of the broker should match the current queue url of roundRobin + // and thus queues are being utilized in round-robin fashion + qURL := testAWSSQSBroker.GetQueueURLForTest(w) + assert.Equal(t, qURL, testAWSSQSBroker.GetCustomQueueURL(rr.Peek())) + } +} diff --git a/v1/config/config.go b/v1/config/config.go index 5e27abd61..05f222267 100644 --- a/v1/config/config.go +++ b/v1/config/config.go @@ -7,9 +7,12 @@ import ( "time" "cloud.google.com/go/pubsub" + "go.mongodb.org/mongo-driver/mongo" + "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/sqs" - "go.mongodb.org/mongo-driver/mongo" + + sqsv2 "github.com/aws/aws-sdk-go-v2/service/sqs" ) const ( @@ -95,7 +98,8 @@ type DynamoDBConfig struct { // SQSConfig wraps SQS related configuration type SQSConfig struct { - Client *sqs.SQS + ClientV1 *sqs.SQS + ClientV2 *sqsv2.Client WaitTimeSeconds int `yaml:"receive_wait_time_seconds" envconfig:"SQS_WAIT_TIME_SECONDS"` // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html // visibility timeout should default to nil to use the overall visibility timeout for the queue diff --git a/v1/factories.go b/v1/factories.go index 60020c87f..02f2dc79c 100644 --- a/v1/factories.go +++ b/v1/factories.go @@ -16,6 +16,7 @@ import ( brokeriface "github.com/RichardKnop/machinery/v1/brokers/iface" redisbroker "github.com/RichardKnop/machinery/v1/brokers/redis" sqsbroker "github.com/RichardKnop/machinery/v1/brokers/sqs" + sqsbrokerv2 "github.com/RichardKnop/machinery/v1/brokers/sqsv2" amqpbackend "github.com/RichardKnop/machinery/v1/backends/amqp" dynamobackend "github.com/RichardKnop/machinery/v1/backends/dynamodb" @@ -85,11 +86,19 @@ func BrokerFactory(cnf *config.Config) (brokeriface.Broker, error) { //even when disabling strict SQS naming check, make sure its still a valid http URL if strings.HasPrefix(cnf.Broker, "https://") || strings.HasPrefix(cnf.Broker, "http://") { - return sqsbroker.New(cnf), nil + if cnf.SQS.ClientV2 != nil { + return sqsbrokerv2.New(cnf), nil + } else { + return sqsbroker.New(cnf), nil + } } } else { if strings.HasPrefix(cnf.Broker, "https://sqs") { - return sqsbroker.New(cnf), nil + if cnf.SQS.ClientV2 != nil { + return sqsbrokerv2.New(cnf), nil + } else { + return sqsbroker.New(cnf), nil + } } } @@ -300,7 +309,7 @@ func ParseRedisDlqURL(url string) (addrs []string, password string, db int, err } else { db, err = strconv.Atoi(parts[1]) if err != nil { - err = fmt.Errorf( "could not extract db from redis+dlq URI: %s", err.Error()) + err = fmt.Errorf("could not extract db from redis+dlq URI: %s", err.Error()) return } dbs[db] = struct{}{} From 94f76a0a979c164726059af5772c684994602a01 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Wed, 19 Oct 2022 23:54:40 +0500 Subject: [PATCH 12/22] Review compliance --- v1/brokers/sqs/sqs.go | 4 ++-- v1/config/config.go | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/v1/brokers/sqs/sqs.go b/v1/brokers/sqs/sqs.go index 56084ba06..d674a5c6f 100644 --- a/v1/brokers/sqs/sqs.go +++ b/v1/brokers/sqs/sqs.go @@ -48,9 +48,9 @@ type ReceivedMessages struct { // New creates new Broker instance func New(cnf *config.Config) iface.Broker { b := &Broker{Broker: common.NewBroker(cnf)} - if cnf.SQS != nil && cnf.SQS.ClientV1 != nil { + if cnf.SQS != nil && cnf.SQS.Client != nil { // Use provided *SQS client - b.service = cnf.SQS.ClientV1 + b.service = cnf.SQS.Client } else { // Initialize a session that the SDK will use to load credentials from the shared credentials file, ~/.aws/credentials. // See details on: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html diff --git a/v1/config/config.go b/v1/config/config.go index 05f222267..6a059a011 100644 --- a/v1/config/config.go +++ b/v1/config/config.go @@ -98,7 +98,13 @@ type DynamoDBConfig struct { // SQSConfig wraps SQS related configuration type SQSConfig struct { - ClientV1 *sqs.SQS + Client *sqs.SQS + // aws-sdk-go-v2 is incompatible with aws-sdk-go + // obtaining credentials fails when running aws-sdk-go IMDSv2 enabled instances + // There is no common interface between v1 and v2 SQS client, v2 requires following + // - requires context as additional parameter on all calls, + // - various parameters of type []*string are changed to []string, similarly map[string]string + // - unlike v1 client which confirms to sqsiface.SQSAPI, v2 client doesn't confirm to a single interface ClientV2 *sqsv2.Client WaitTimeSeconds int `yaml:"receive_wait_time_seconds" envconfig:"SQS_WAIT_TIME_SECONDS"` // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html From ca5830db1f84099eb9f7726b150de456258a8cbb Mon Sep 17 00:00:00 2001 From: Ramanarayana Narmala <3530406+suvera@users.noreply.github.com> Date: Fri, 30 Dec 2022 18:49:15 +0530 Subject: [PATCH 13/22] Support ReceiveCount attribute in the sqs consumer Support ReceiveCount attribute in the sqs consumer --- v1/brokers/sqs/sqs.go | 4 +++- v1/tasks/signature.go | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/v1/brokers/sqs/sqs.go b/v1/brokers/sqs/sqs.go index d674a5c6f..89e8ab3f1 100644 --- a/v1/brokers/sqs/sqs.go +++ b/v1/brokers/sqs/sqs.go @@ -232,6 +232,8 @@ func (b *Broker) consumeOne(sqsReceivedMsgs *ReceivedMessages, taskProcessor ifa queueName := b.urlToQueue(*(sqsReceivedMsgs.queue)) sig.RoutingKey = *queueName } + + sig.Attributes = delivery.Messages[0].Attributes err := taskProcessor.Process(sig) if err != nil { @@ -280,7 +282,7 @@ func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, err } input := &awssqs.ReceiveMessageInput{ AttributeNames: []*string{ - aws.String(awssqs.MessageSystemAttributeNameSentTimestamp), + aws.String(awssqs.QueueAttributeNameAll), }, MessageAttributeNames: []*string{ aws.String(awssqs.QueueAttributeNameAll), diff --git a/v1/tasks/signature.go b/v1/tasks/signature.go index c05eff3d0..18382d11e 100644 --- a/v1/tasks/signature.go +++ b/v1/tasks/signature.go @@ -68,6 +68,7 @@ type Signature struct { // IgnoreWhenTaskNotRegistered auto removes the request when there is no handeler available // When this is true a task with no handler will be ignored and not placed back in the queue IgnoreWhenTaskNotRegistered bool + Attributes map[string]*string } // NewSignature creates a new task signature From ddd86e7296d246ccc6054308602b1ac8d8b7ff47 Mon Sep 17 00:00:00 2001 From: Ramanarayana Narmala <3530406+suvera@users.noreply.github.com> Date: Tue, 3 Jan 2023 09:42:53 +0530 Subject: [PATCH 14/22] Address the review comments --- v1/brokers/sqs/sqs.go | 4 +++- v1/tasks/signature.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/v1/brokers/sqs/sqs.go b/v1/brokers/sqs/sqs.go index 89e8ab3f1..26dbc8773 100644 --- a/v1/brokers/sqs/sqs.go +++ b/v1/brokers/sqs/sqs.go @@ -24,6 +24,8 @@ import ( const ( maxAWSSQSDelay = time.Minute * 15 // Max supported SQS delay is 15 min: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html + // Returns all values, A list of attributes that need to be returned along with each message + messageSystemAttributeNameAll = "All" ) // Broker represents a AWS SQS broker @@ -282,7 +284,7 @@ func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, err } input := &awssqs.ReceiveMessageInput{ AttributeNames: []*string{ - aws.String(awssqs.QueueAttributeNameAll), + aws.String(messageSystemAttributeNameAll), }, MessageAttributeNames: []*string{ aws.String(awssqs.QueueAttributeNameAll), diff --git a/v1/tasks/signature.go b/v1/tasks/signature.go index 18382d11e..f547e52b0 100644 --- a/v1/tasks/signature.go +++ b/v1/tasks/signature.go @@ -62,12 +62,14 @@ type Signature struct { BrokerMessageGroupId string //ReceiptHandle of SQS Message SQSReceiptHandle string - // StopTaskDeletionOnError used with sqs when we want to send failed messages to dlq, - // and don't want machinery to delete from source queue + // StopTaskDeletionOnError used with sqs when we want to send failed messages to dlq, + // and don't want machinery to delete from source queue StopTaskDeletionOnError bool // IgnoreWhenTaskNotRegistered auto removes the request when there is no handeler available // When this is true a task with no handler will be ignored and not placed back in the queue IgnoreWhenTaskNotRegistered bool + // A map of the attributes requested in ReceiveMessage, and their respective values. e.g. ApproximateReceiveCount + // for SQS, check https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_Message.html#API_Message_Contents Attributes map[string]*string } From b222b2b884ba571412c22683d3b09a5f578eade5 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Mon, 17 Apr 2023 16:39:03 +0500 Subject: [PATCH 15/22] Update go.mongodb.org/mongo-driver to resolve CVE-2021-20329 --- go.mod | 14 +++----------- go.sum | 61 +++++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 70e658ffb..a7165a293 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,7 @@ require ( github.com/btcsuite/btcutil v1.0.2 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/go-redis/redis v6.15.6+incompatible - github.com/go-stack/stack v1.8.0 // indirect github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc // indirect - github.com/golang/snappy v0.0.1 // indirect github.com/gomodule/redigo v2.0.0+incompatible github.com/google/uuid v1.1.1 github.com/jstemmer/go-junit-report v0.9.1 // indirect @@ -23,21 +21,15 @@ require ( github.com/onsi/ginkgo v1.10.0 // indirect github.com/onsi/gomega v1.7.0 // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 - github.com/stretchr/testify v1.3.0 + github.com/stretchr/testify v1.6.1 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 // indirect - github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 // indirect github.com/urfave/cli v1.22.1 - github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect - github.com/xdg/stringprep v1.0.0 // indirect - go.mongodb.org/mongo-driver v1.1.2 + go.mongodb.org/mongo-driver v1.11.4 go.opencensus.io v0.22.1 // indirect golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect - golang.org/x/net v0.0.0-20191007182048-72f939374954 // indirect - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect - golang.org/x/sys v0.0.0-20191008105621-543471e840be // indirect golang.org/x/tools v0.0.0-20191007185444-6536af71d98a // indirect google.golang.org/api v0.11.0 // indirect google.golang.org/appengine v1.6.5 // indirect diff --git a/go.sum b/go.sum index 182ebd79b..4c228e679 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg= github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -98,6 +96,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -128,11 +127,15 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.0 h1:XaTQmdKecIbwNHpzOIy0XMoEG5bmv/n0OVyaF1NKUdo= @@ -142,8 +145,8 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -155,20 +158,24 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmV github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= -github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 h1:BP2bjP495BBPaBcS5rmqviTfrOkN5rO5ceKAMRZCRFc= -github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= @@ -177,8 +184,9 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -211,8 +219,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU= -golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -222,8 +230,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -233,12 +241,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -260,6 +273,7 @@ golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191007185444-6536af71d98a h1:mtF1GhqcFEC1RVSQxvgrZWOM22dax6fiM9VfcQoTv6U= golang.org/x/tools v0.0.0-20191007185444-6536af71d98a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -299,6 +313,9 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From d85f17336ab1d91d68738c03cc2946fcb330a61c Mon Sep 17 00:00:00 2001 From: Umar Naveed Date: Thu, 27 Apr 2023 20:05:59 +0500 Subject: [PATCH 16/22] add receive count to redis-dlq broker --- v1/brokers/redis/goredis-dlq.go | 41 ++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index a0f5ca634..ff6f1d0c1 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -26,6 +26,7 @@ const ( messageVisibilitySet = "message-visibility-set" hSetMessageKey = "message" hSetQueueKey = "queue" + hSetRetryKey = "visibility_counter" taskPrefix = "task_%s" ) @@ -47,19 +48,19 @@ func NewGR_DLQ(cnf *config.Config, addrs []string, password string, db int) ifac b := &BrokerGR_DLQ{Broker: common.NewBroker(cnf)} ropt := &redis.UniversalOptions{ - Addrs: addrs, - DB: db, - Password: password, - ReadTimeout: time.Duration(cnf.Redis.ReadTimeout) * time.Second, - WriteTimeout: time.Duration(cnf.Redis.WriteTimeout) * time.Second, - DialTimeout: time.Duration(cnf.Redis.ConnectTimeout) * time.Second, - IdleTimeout: time.Duration(cnf.Redis.IdleTimeout) * time.Second, - MinIdleConns: cnf.Redis.MinIdleConns, + Addrs: addrs, + DB: db, + Password: password, + ReadTimeout: time.Duration(cnf.Redis.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(cnf.Redis.WriteTimeout) * time.Second, + DialTimeout: time.Duration(cnf.Redis.ConnectTimeout) * time.Second, + IdleTimeout: time.Duration(cnf.Redis.IdleTimeout) * time.Second, + MinIdleConns: cnf.Redis.MinIdleConns, MinRetryBackoff: time.Duration(cnf.Redis.MinRetryBackoff) * time.Millisecond, MaxRetryBackoff: time.Duration(cnf.Redis.MaxRetryBackoff) * time.Millisecond, - MaxRetries: cnf.Redis.MaxRetries, - PoolSize: cnf.Redis.PoolSize, - TLSConfig: cnf.TLSConfig, + MaxRetries: cnf.Redis.MaxRetries, + PoolSize: cnf.Redis.PoolSize, + TLSConfig: cnf.TLSConfig, } if cnf.Redis != nil { // if we're specifying MasterName here, then we'll always connect to db 0, since provided db is ignored in cluster mode @@ -313,6 +314,24 @@ func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProce return nil } + stringCmd := b.rclient.HGet(gHash, hSetRetryKey) + if err := stringCmd.Err(); err != nil { + log.ERROR.Printf("Could not retrieve message keys from redis. Error: %s", err.Error()) + } + val := stringCmd.Val() + + receiveCount := val + if val == "" { + receiveCount = "1" + boolCmd := b.rclient.HSet(gHash, hSetRetryKey, 1) + if err := boolCmd.Err(); err != nil { + log.ERROR.Printf("Could not set retry count for message in HSet. Error: %s", err.Error()) + } + } + + signature.Attributes = map[string]*string{} + signature.Attributes["ApproximateReceiveCount"] = &receiveCount + log.DEBUG.Printf("Received new message: %+v", signature) log.INFO.Printf("Processing task. Old UUID: %s New UUID: %s", oldUuid, signature.UUID) From 2e016bc28b2c60dbebe5f8220cb95ec1fef534e4 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Mon, 8 May 2023 18:19:19 +0500 Subject: [PATCH 17/22] golang.org/x/net to fix vulns --- go.mod | 4 ++-- go.sum | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index a7165a293..7a658f1f2 100644 --- a/go.mod +++ b/go.mod @@ -26,11 +26,11 @@ require ( github.com/stretchr/testify v1.6.1 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 // indirect github.com/urfave/cli v1.22.1 - go.mongodb.org/mongo-driver v1.11.4 + go.mongodb.org/mongo-driver v1.11.6 go.opencensus.io v0.22.1 // indirect golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 // indirect golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect - golang.org/x/tools v0.0.0-20191007185444-6536af71d98a // indirect + golang.org/x/net v0.9.0 // indirect google.golang.org/api v0.11.0 // indirect google.golang.org/appengine v1.6.5 // indirect google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect diff --git a/go.sum b/go.sum index 4c228e679..7a9ec1321 100644 --- a/go.sum +++ b/go.sum @@ -174,8 +174,9 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= -go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o= +go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= @@ -185,6 +186,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -208,6 +210,9 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -219,8 +224,12 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -230,8 +239,10 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -243,15 +254,25 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -270,8 +291,10 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190917162342-3b4f30a44f3b/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191007185444-6536af71d98a h1:mtF1GhqcFEC1RVSQxvgrZWOM22dax6fiM9VfcQoTv6U= -golang.org/x/tools v0.0.0-20191007185444-6536af71d98a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= From 87533015cd12e78cbb45dd9b3b67bd7aef4869b8 Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Wed, 10 May 2023 11:34:25 +0500 Subject: [PATCH 18/22] Port changes from https://github.com/securitiai/machinery/pull/11 --- v1/brokers/sqsv2/sqs.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/v1/brokers/sqsv2/sqs.go b/v1/brokers/sqsv2/sqs.go index cd096a60d..ea35f486b 100644 --- a/v1/brokers/sqsv2/sqs.go +++ b/v1/brokers/sqsv2/sqs.go @@ -255,6 +255,13 @@ func (b *Broker) consumeOne(sqsReceivedMsgs *ReceivedMessages, taskProcessor ifa sig.RoutingKey = *queueName } + // Port changes from https://github.com/securitiai/machinery/pull/11 to sqs v2 + var attrs map[string]*string + for key, val := range delivery.Messages[0].Attributes { + attrs[key] = aws.String(val) + } + sig.Attributes = attrs + err := taskProcessor.Process(sig) if err != nil { // stop task deletion in case we want to send messages to dlq in sqs @@ -302,7 +309,7 @@ func (b *Broker) receiveMessage(qURL *string) (*awssqs.ReceiveMessageOutput, err } input := &awssqs.ReceiveMessageInput{ AttributeNames: []types.QueueAttributeName{ - types.QueueAttributeName(types.MessageSystemAttributeNameSentTimestamp), + types.QueueAttributeNameAll, }, MessageAttributeNames: []string{ string(types.QueueAttributeNameAll), From 4f347cc54a77e86588c89140d5aeeba5ca9d0e59 Mon Sep 17 00:00:00 2001 From: Umar Naveed Date: Tue, 16 May 2023 12:44:47 +0500 Subject: [PATCH 19/22] increment receiveCount before adding to signature --- go.mod | 1 + go.sum | 15 +++++++++++++-- v1/brokers/redis/goredis-dlq.go | 16 +++++++--------- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 70e658ffb..e415a4cb0 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/onsi/gomega v1.7.0 // indirect github.com/opentracing/opentracing-go v1.1.0 github.com/pkg/errors v0.8.1 + github.com/spf13/cast v1.5.0 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 github.com/stretchr/testify v1.3.0 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 // indirect diff --git a/go.sum b/go.sum index 182ebd79b..27a5ab252 100644 --- a/go.sum +++ b/go.sum @@ -67,10 +67,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -98,6 +101,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -128,11 +132,13 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.0 h1:XaTQmdKecIbwNHpzOIy0XMoEG5bmv/n0OVyaF1NKUdo= @@ -147,10 +153,14 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -260,6 +270,7 @@ golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191007185444-6536af71d98a h1:mtF1GhqcFEC1RVSQxvgrZWOM22dax6fiM9VfcQoTv6U= golang.org/x/tools v0.0.0-20191007185444-6536af71d98a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/v1/brokers/redis/goredis-dlq.go b/v1/brokers/redis/goredis-dlq.go index ff6f1d0c1..b38049470 100644 --- a/v1/brokers/redis/goredis-dlq.go +++ b/v1/brokers/redis/goredis-dlq.go @@ -20,6 +20,7 @@ import ( "github.com/RichardKnop/redsync" "github.com/btcsuite/btcutil/base58" "github.com/go-redis/redis" + "github.com/spf13/cast" ) const ( @@ -320,17 +321,14 @@ func (b *BrokerGR_DLQ) consumeOne(delivery []byte, taskProcessor iface.TaskProce } val := stringCmd.Val() - receiveCount := val - if val == "" { - receiveCount = "1" - boolCmd := b.rclient.HSet(gHash, hSetRetryKey, 1) - if err := boolCmd.Err(); err != nil { - log.ERROR.Printf("Could not set retry count for message in HSet. Error: %s", err.Error()) - } - } + receiveCount := cast.ToInt(val) + //increment before adding to signature since ApproximateReceiveCount is the number of times a message is received, whereas the value stored in hset represents the number of retries which is one less than ApproximateReceiveCount + receiveCount++ + + receiveCountString := cast.ToString(receiveCount) signature.Attributes = map[string]*string{} - signature.Attributes["ApproximateReceiveCount"] = &receiveCount + signature.Attributes["ApproximateReceiveCount"] = &receiveCountString log.DEBUG.Printf("Received new message: %+v", signature) log.INFO.Printf("Processing task. Old UUID: %s New UUID: %s", oldUuid, signature.UUID) From 98e441b1fa520ec760dee444d82379a214f85d77 Mon Sep 17 00:00:00 2001 From: Umar Naveed Date: Tue, 16 May 2023 12:48:40 +0500 Subject: [PATCH 20/22] resolve merge conflicts --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e415a4cb0..eb6818d09 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/onsi/ginkgo v1.10.0 // indirect github.com/onsi/gomega v1.7.0 // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pkg/errors v0.8.1 + github.com/pkg/errors v0.9.1 github.com/spf13/cast v1.5.0 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 github.com/stretchr/testify v1.3.0 diff --git a/go.sum b/go.sum index 27a5ab252..8fffa3ac4 100644 --- a/go.sum +++ b/go.sum @@ -148,8 +148,8 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= From 98312034cc103d1b2af9ef759dbbffd38e3a07a9 Mon Sep 17 00:00:00 2001 From: Umar Naveed Date: Tue, 16 May 2023 12:58:40 +0500 Subject: [PATCH 21/22] sync dependencies --- go.sum | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.sum b/go.sum index f78e59750..da226174c 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -132,7 +133,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= From 8af9ab63703715fdc14667aa625cadef3bb46eee Mon Sep 17 00:00:00 2001 From: Muhammad Ali Date: Thu, 25 May 2023 19:20:07 +0500 Subject: [PATCH 22/22] Fix map initialization when copying message attributes --- v1/brokers/sqsv2/sqs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1/brokers/sqsv2/sqs.go b/v1/brokers/sqsv2/sqs.go index ea35f486b..160062a39 100644 --- a/v1/brokers/sqsv2/sqs.go +++ b/v1/brokers/sqsv2/sqs.go @@ -256,7 +256,7 @@ func (b *Broker) consumeOne(sqsReceivedMsgs *ReceivedMessages, taskProcessor ifa } // Port changes from https://github.com/securitiai/machinery/pull/11 to sqs v2 - var attrs map[string]*string + var attrs = map[string]*string{} for key, val := range delivery.Messages[0].Attributes { attrs[key] = aws.String(val) }