From 0bcbe193be28664bb789704222823fabcd8704fa Mon Sep 17 00:00:00 2001 From: amirylm Date: Fri, 23 Feb 2024 00:40:49 +0200 Subject: [PATCH] check multiple responses: - server timeout - unauthorized - bad req - internal server err - not found --- .../plugins/ocr2keeper/integration_21_test.go | 95 +++++++++++-------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go b/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go index 6155e362093..098398e66d8 100644 --- a/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go @@ -2,14 +2,16 @@ package ocr2keeper_test import ( "context" - "crypto/rand" + crand "crypto/rand" "encoding/hex" "encoding/json" "fmt" "math/big" + "math/rand" "net/http" "strings" "sync" + "sync/atomic" "testing" "time" @@ -60,7 +62,7 @@ import ( func TestFilterNamesFromSpec21(t *testing.T) { b := make([]byte, 20) - _, err := rand.Read(b) + _, err := crand.Read(b) require.NoError(t, err) address := common.HexToAddress(hexutil.Encode(b)) @@ -415,48 +417,39 @@ func TestIntegration_KeeperPluginLogUpkeep_ErrHandler(t *testing.T) { _, mercuryServer := setupNodes(t, nodeKeys, registry, backend, registryOwner) - const upkeepCount = 10 - const mercuryFailRounds = 7 - const mercuryFailCount = upkeepCount * 3 * mercuryFailRounds - - // testing with the mercury server involves mocking responses. currently, - // there is not a way to connect a mercury call to an upkeep id (though we - // could add custom headers) so the test must be fairly basic and just - // count calls before switching to successes - var ( - mu sync.Mutex - count int - ) - - mercuryServer.RegisterHandler(func(w http.ResponseWriter, r *http.Request) { - mu.Lock() - defer mu.Unlock() - - count++ - - _ = r.ParseForm() - - t.Logf("MercuryHTTPServe:RequestURI: %s", r.RequestURI) + upkeepCount := 10 - for key, value := range r.Form { - t.Logf("MercuryHTTPServe:FormValue: key: %s; value: %s;", key, value) - } - - // the streams lookup retries against the remote server 3 times before - // returning a result as retryable. - // the simulation here should force the streams lookup process to return - // retryable 2 times. - // the total count of failures should be (upkeepCount * 3 * tryCount) - if count <= mercuryFailCount { - w.WriteHeader(http.StatusNotFound) - return + respTimeout := -1 + errResponses := []int{ + respTimeout, + http.StatusUnauthorized, + http.StatusBadRequest, + http.StatusInternalServerError, + } + // inline retries (3) * retry rounds (6) + timeout round (1) + mercuryFailCountPerUpkeep := 3*6 + 1 + // mercuryFailCount is the number of times the mercury server will return an + // error before returning a success + mercuryFailCount := 2 + (mercuryFailCountPerUpkeep * 3) + for i := 3; i < mercuryFailCount; i++ { + errResponses = append(errResponses, http.StatusNotFound) + } + startMercuryServer(t, mercuryServer, func(i int) (int, []byte) { + if i < len(errResponses) { + resp := errResponses[i] + switch resp { + case http.StatusNotFound, http.StatusInternalServerError: + // in case we got a 404 or 500, we wait a bit to simulate real behavior + time.Sleep(time.Duration(rand.Intn(2500)) * time.Millisecond) + case respTimeout: // mercury server timeout + time.Sleep(60 * time.Second) // 60s is the default timeout + resp = http.StatusNotFound + default: + } + return resp, nil } - - // start sending success messages output := `{"chainlinkBlob":"0x0001c38d71fed6c320b90e84b6f559459814d068e2a1700adc931ca9717d4fe70000000000000000000000000000000000000000000000000000000001a80b52b4bf1233f9cb71144a253a1791b202113c4ab4a92fa1b176d684b4959666ff8200000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000260000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001004254432d5553442d415242495452554d2d544553544e4554000000000000000000000000000000000000000000000000000000000000000000000000645570be000000000000000000000000000000000000000000000000000002af2b818dc5000000000000000000000000000000000000000000000000000002af2426faf3000000000000000000000000000000000000000000000000000002af32dc209700000000000000000000000000000000000000000000000000000000012130f8df0a9745bb6ad5e2df605e158ba8ad8a33ef8a0acf9851f0f01668a3a3f2b68600000000000000000000000000000000000000000000000000000000012130f60000000000000000000000000000000000000000000000000000000000000002c4a7958dce105089cf5edb68dad7dcfe8618d7784eb397f97d5a5fade78c11a58275aebda478968e545f7e3657aba9dcbe8d44605e4c6fde3e24edd5e22c94270000000000000000000000000000000000000000000000000000000000000002459c12d33986018a8959566d145225f0c4a4e61a9a3f50361ccff397899314f0018162cf10cd89897635a0bb62a822355bd199d09f4abe76e4d05261bb44733d"}` - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(output)) + return http.StatusOK, []byte(output) }) defer mercuryServer.Stop() @@ -508,6 +501,25 @@ func TestIntegration_KeeperPluginLogUpkeep_ErrHandler(t *testing.T) { done() } +func startMercuryServer(t *testing.T, mercuryServer *SimulatedMercuryServer, responder func(i int) (int, []byte)) { + i := atomic.Int32{} + mercuryServer.RegisterHandler(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + t.Logf("MercuryHTTPServe:RequestURI: %s", r.RequestURI) + for key, value := range r.Form { + t.Logf("MercuryHTTPServe:FormValue: key: %s; value: %s;", key, value) + } + + ii := int(i.Load()) + i.Add(1) + status, body := responder(ii) + w.WriteHeader(status) + if len(body) > 0 { + _, _ = w.Write(body) + } + }) +} + func listenEvents(t *testing.T, backend *backends.SimulatedBackend, addrs []common.Address, topic common.Hash, startBlock int64, count int) (func() bool, func()) { ctx, cancel := context.WithCancel(testutils.Context(t)) visited := make(map[string]bool) @@ -536,6 +548,7 @@ func listenEvents(t *testing.T, backend *backends.SimulatedBackend, addrs []comm continue } visited[visitedID] = true + t.Logf("found err handler log in block %d [%s]", log.BlockNumber, log.BlockHash.Hex()) cacheID := log.Address.Hex() count, ok := cache.Load(cacheID) if !ok {