Skip to content

Commit

Permalink
Use atomics to make writing of results safe in removeObjectsOneByOne
Browse files Browse the repository at this point in the history
Previous implementation didn't have any synchronization mechanism for
goroutines that does the work.

There are multiple approaches to make it work correctly, let's start with
the simplest - atomics.
  • Loading branch information
a-narsudinov committed Oct 27, 2023
1 parent da3638e commit d3e89f1
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions pkg/s3/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"fmt"
"net/url"
"sync/atomic"

"github.com/golang/glog"
"github.com/minio/minio-go/v7"
Expand Down Expand Up @@ -173,8 +174,8 @@ func (client *s3Client) removeObjectsOneByOne(bucketName, prefix string) error {
objectsCh := make(chan minio.ObjectInfo, 1)
guardCh := make(chan int, parallelism)
var listErr error
totalObjects := 0
removeErrors := 0
var totalObjects int64 = 0
var removeErrors int64 = 0

go func() {
defer close(objectsCh)
Expand All @@ -185,7 +186,7 @@ func (client *s3Client) removeObjectsOneByOne(bucketName, prefix string) error {
listErr = object.Err
return
}
totalObjects++
atomic.AddInt64(&totalObjects, 1)
objectsCh <- object
}
}()
Expand All @@ -202,7 +203,7 @@ func (client *s3Client) removeObjectsOneByOne(bucketName, prefix string) error {
minio.RemoveObjectOptions{VersionID: obj.VersionID})
if err != nil {
glog.Errorf("Failed to remove object %s, error: %s", obj.Key, err)
removeErrors++
atomic.AddInt64(&removeErrors, 1)
}
<- guardCh
}(object)
Expand Down

0 comments on commit d3e89f1

Please sign in to comment.