Skip to content

Commit

Permalink
adds trace log messages to LR
Browse files Browse the repository at this point in the history
to troubleshoot potential deadlock
  • Loading branch information
vroldanbet committed Mar 26, 2024
1 parent 98b04e6 commit d9ca2e6
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 0 deletions.
7 changes: 7 additions & 0 deletions internal/graph/checkingresourcestream.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package graph
import (
"cmp"
"context"
log "github.com/authzed/spicedb/internal/logging"
"slices"
"sync"

Expand Down Expand Up @@ -279,21 +280,26 @@ func newCheckingResourceStream(
// waitForPublishing waits for the publishing goroutine to complete its work, and returns the number
// of published *reachable* resources or the error that occurred during checking or publishing.
func (crs *checkingResourceStream) waitForPublishing() (uint64, *v1.Cursor, error) {

log.Trace().Msg("marking reachable resources completed")
// Mark that no new items will come in from the reachable resources stream.
for i := 0; i < int(crs.concurrencyLimit); i++ {
crs.reachableResourcesCompleted <- struct{}{}
}

// Wait for all existing processing to complete.
log.Trace().Msg("waiting for all existing processing to complete")
crs.processingWaitGroup.Wait()

// Run a final processing call to ensure there are no remaining items.
log.Trace().Msg("making sure there are no remaining items in the checking stream")
_, err := crs.runProcess(true)
if err != nil {
return 0, nil, err
}

// Mark publishing as ready for final publishing.
log.Trace().Msg("marking publishing as ready for final publishing")
select {
case crs.availableForPublishing <- false:
break
Expand All @@ -304,6 +310,7 @@ func (crs *checkingResourceStream) waitForPublishing() (uint64, *v1.Cursor, erro
}

// Wait for any remaining publishing to complete.
log.Trace().Msg("waiting for any remaining publishing to complete")
crs.publishingWaitGroup.Wait()

return crs.reachableResourcesCount, crs.lastReachableResourceCursor, crs.err
Expand Down
3 changes: 3 additions & 0 deletions internal/graph/lookupresources.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package graph
import (
"context"
"errors"
log "github.com/authzed/spicedb/internal/logging"

"github.com/authzed/spicedb/internal/dispatch"
"github.com/authzed/spicedb/pkg/datastore"
Expand Down Expand Up @@ -77,11 +78,13 @@ func (cl *CursoredLookupResources) LookupResources(
}
}

log.Trace().Msg("waiting for checking stream publishing")
reachableCount, newCursor, err := checkingStream.waitForPublishing()
if err != nil {
return err
}

log.Trace().Uint64("reachable-count", reachableCount).Msg("completed checking stream publishing")
reachableResourcesCursor = newCursor
if reachableCount == 0 {
return nil
Expand Down

0 comments on commit d9ca2e6

Please sign in to comment.