Add delay between pull request updates

In projects with many open PRs using the automatic updates feature,
Bulldozer can trigger GitHub's secondary rate limits by updating (and
possibly querying) pull requests too quickly. To avoid this, add some
delay between operations.

For queries, use a fixed 250ms delay for now. This shouldn't slow things
down too much, but does prevent Bulldozer from blasting through PRs as
fast as possible.

For updates, use an exponential delay that starts at 1s and maxes out at
60s. I believe the secondary limits (at least for GitHub Enterprise) are
computed over a minute, so this will limit us to one update per minute
for large batches. I don't think this will be prohibitively slow, but we
could explore making this configurable in the future.

As part of this, refactor the logic so that we don't block queries on
previous updates.

server/handler/push.go

@@ -17,13 +17,23 @@ package handler
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	"github.com/google/go-github/v40/github"
+	"github.com/palantir/bulldozer/bulldozer"
 	"github.com/palantir/bulldozer/pull"
 	"github.com/palantir/go-githubapp/githubapp"
 	"github.com/pkg/errors"
 )
 
+const (
+	PullQueryDelay = 250 * time.Millisecond
+
+	PullUpdateBaseDelay = 1 * time.Second
+	PullUpdateMaxDelay  = 60 * time.Second
+	PullUpdateDelayMult = 1.5
+)
+
 type Push struct {
 	Base
 }
@@ -75,18 +85,64 @@ func (h *Push) Handle(ctx context.Context, eventType, deliveryID string, payload
 		return err
 	}
 
-	for _, pr := range prs {
+	if config == nil {
+		logger.Debug().Msg("Skipping pull request updates to missing configuration")
+		return nil
+	}
+
+	var toUpdate []updateCtx
+	for i, pr := range prs {
 		logger := logger.With().Int(githubapp.LogKeyPRNum, pr.GetNumber()).Logger()
-		logger.Debug().Msgf("Considering pull request for update")
+		logger.Debug().Msg("Checking if pull request should update")
 
+		ctx := logger.WithContext(ctx)
 		pullCtx := pull.NewGithubContext(client, pr)
-		if _, err := h.UpdatePullRequest(logger.WithContext(ctx), pullCtx, client, config, pr, baseRef); err != nil {
-			logger.Error().Err(errors.WithStack(err)).Msg("Error updating pull request")
+
+		shouldUpdate, err := bulldozer.ShouldUpdatePR(ctx, pullCtx, config.Update)
+		if err != nil {
+			logger.Error().Err(err).Msg("Error determining if pull request should update, skipping")
+			continue
+		}
+		if shouldUpdate {
+			toUpdate = append(toUpdate, updateCtx{
+				ctx:     ctx,
+				pullCtx: pullCtx,
+			})
+		}
+
+		if i < len(prs)-1 {
+			time.Sleep(delay(i, PullQueryDelay, 1, PullQueryDelay))
+		}
+	}
+
+	logger.Info().Msgf("Found %d pull requests that need updates", len(toUpdate))
+	for i, pr := range toUpdate {
+		bulldozer.UpdatePR(pr.ctx, pr.pullCtx, client, config.Update, baseRef)
+		if i < len(toUpdate)-1 {
+			d := delay(i, PullUpdateBaseDelay, PullUpdateDelayMult, PullUpdateMaxDelay)
+			logger.Debug().Msgf("Waiting %v until next update to avoid GitHub rate limits", d)
+			time.Sleep(d)
 		}
 	}
 
 	return nil
 }
 
+type updateCtx struct {
+	ctx     context.Context
+	pullCtx pull.Context
+}
+
+func delay(iter int, base time.Duration, mult float64, max time.Duration) time.Duration {
+	t := base
+	for ; iter > 0; iter-- {
+		t = time.Duration(mult * float64(t))
+		if t > max {
+			return max
+		}
+	}
+	return t
+}
+
 // type assertion
 var _ githubapp.EventHandler = &Push{}