From e4b610f40c4811f1065f5b10cb8975c200e92148 Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Wed, 11 Sep 2024 17:22:30 +0200 Subject: [PATCH 1/3] hotfix v3.1.5-near --- .../lavasession/consumer_session_manager.go | 3 ++- .../consumer_session_manager_test.go | 22 +++++++++---------- .../end_to_end_lavasession_test.go | 4 ++-- .../lavasession/single_consumer_session.go | 8 ++++--- .../consumer_relay_state_machine.go | 21 +++++++++++++++++- .../consumer_relay_state_machine_test.go | 4 ++++ protocol/rpcconsumer/rpcconsumer_server.go | 11 +++++++++- 7 files changed, 54 insertions(+), 19 deletions(-) diff --git a/protocol/lavasession/consumer_session_manager.go b/protocol/lavasession/consumer_session_manager.go index fca5f44afb..05a7a2adad 100644 --- a/protocol/lavasession/consumer_session_manager.go +++ b/protocol/lavasession/consumer_session_manager.go @@ -1017,6 +1017,7 @@ func (csm *ConsumerSessionManager) OnSessionDone( numOfProviders int, providersCount uint64, isHangingApi bool, + reduceAvailability bool, ) error { // release locks, update CU, relaynum etc.. if err := consumerSession.VerifyLock(); err != nil { @@ -1039,7 +1040,7 @@ func (csm *ConsumerSessionManager) OnSessionDone( consumerSession.ConsecutiveErrors = []error{} consumerSession.LatestBlock = latestServicedBlock // update latest serviced block // calculate QoS - consumerSession.CalculateQoS(currentLatency, expectedLatency, expectedBH-latestServicedBlock, numOfProviders, int64(providersCount)) + consumerSession.CalculateQoS(currentLatency, expectedLatency, expectedBH-latestServicedBlock, numOfProviders, int64(providersCount), reduceAvailability) go csm.providerOptimizer.AppendRelayData(consumerSession.Parent.PublicLavaAddress, currentLatency, isHangingApi, specComputeUnits, uint64(latestServicedBlock)) csm.updateMetricsManager(consumerSession, currentLatency, !isHangingApi) // apply latency only for non hanging apis return nil diff --git a/protocol/lavasession/consumer_session_manager_test.go b/protocol/lavasession/consumer_session_manager_test.go index b22f0c2e61..178cee24dc 100644 --- a/protocol/lavasession/consumer_session_manager_test.go +++ b/protocol/lavasession/consumer_session_manager_test.go @@ -83,7 +83,7 @@ func TestHappyFlow(t *testing.T) { require.NotNil(t, cs) require.Equal(t, cs.Epoch, csm.currentEpoch) require.Equal(t, cs.Session.LatestRelayCu, cuForFirstRequest) - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, cs.Session.CuSum, cuForFirstRequest) require.Equal(t, cs.Session.LatestRelayCu, latestRelayCuAfterDone) @@ -416,7 +416,7 @@ func runOnSessionDoneForConsumerSessionMap(t *testing.T, css ConsumerSessionsMap require.NotNil(t, cs) require.Equal(t, cs.Epoch, csm.currentEpoch) require.Equal(t, cs.Session.LatestRelayCu, cuForFirstRequest) - err := csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err := csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, cs.Session.CuSum, cuForFirstRequest) require.Equal(t, cs.Session.LatestRelayCu, latestRelayCuAfterDone) @@ -448,7 +448,7 @@ func TestHappyFlowVirtualEpoch(t *testing.T) { require.NotNil(t, cs) require.Equal(t, cs.Epoch, csm.currentEpoch) require.Equal(t, cs.Session.LatestRelayCu, maxCuForVirtualEpoch*(virtualEpoch+1)) - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, maxCuForVirtualEpoch*(virtualEpoch+1), time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, maxCuForVirtualEpoch*(virtualEpoch+1), time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, cs.Session.CuSum, maxCuForVirtualEpoch*(virtualEpoch+1)) require.Equal(t, cs.Session.LatestRelayCu, latestRelayCuAfterDone) @@ -484,7 +484,7 @@ func TestPairingReset(t *testing.T) { require.NotNil(t, cs) require.Equal(t, cs.Epoch, csm.currentEpoch) require.Equal(t, cs.Session.LatestRelayCu, cuForFirstRequest) - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, cs.Session.CuSum, cuForFirstRequest) require.Equal(t, cs.Session.LatestRelayCu, latestRelayCuAfterDone) @@ -573,7 +573,7 @@ func TestPairingResetWithMultipleFailures(t *testing.T) { require.NotNil(t, cs) require.Equal(t, cs.Epoch, csm.currentEpoch) require.Equal(t, cs.Session.LatestRelayCu, cuForFirstRequest) - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, cs.Session.CuSum, cuForFirstRequest) require.Equal(t, cs.Session.LatestRelayCu, latestRelayCuAfterDone) @@ -619,7 +619,7 @@ func TestSuccessAndFailureOfSessionWithUpdatePairingsInTheMiddle(t *testing.T) { require.Equal(t, epoch, csm.currentEpoch) if rand.Intn(2) > 0 { - err = csm.OnSessionDone(cs, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, cs.CuSum, cuForFirstRequest) require.Equal(t, cs.LatestRelayCu, latestRelayCuAfterDone) @@ -653,7 +653,7 @@ func TestSuccessAndFailureOfSessionWithUpdatePairingsInTheMiddle(t *testing.T) { for j := numberOfAllowedSessionsPerConsumer / 2; j < numberOfAllowedSessionsPerConsumer; j++ { cs := sessionList[j].cs if rand.Intn(2) > 0 { - err = csm.OnSessionDone(cs, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) require.Equal(t, sessionListData[j].cuSum+cuForFirstRequest, cs.CuSum) require.Equal(t, cs.LatestRelayCu, latestRelayCuAfterDone) @@ -676,7 +676,7 @@ func successfulSession(ctx context.Context, csm *ConsumerSessionManager, t *test for _, cs := range css { require.NotNil(t, cs) time.Sleep(time.Duration((rand.Intn(500) + 1)) * time.Millisecond) - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) ch <- p } @@ -957,7 +957,7 @@ func TestPairingWithAddons(t *testing.T) { css, err := csm.GetSessions(ctx, cuForFirstRequest, NewUsedProviders(nil), servicedBlockNumber, addon, nil, common.NO_STATE, 0) // get a session require.NoError(t, err) for _, cs := range css { - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) } }) @@ -1032,7 +1032,7 @@ func TestPairingWithExtensions(t *testing.T) { css, err := csm.GetSessions(ctx, cuForFirstRequest, NewUsedProviders(nil), servicedBlockNumber, extensionOpt.addon, extensionsList, common.NO_STATE, 0) // get a session require.NoError(t, err) for _, cs := range css { - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) } }) @@ -1068,7 +1068,7 @@ func TestPairingWithStateful(t *testing.T) { require.NoError(t, err) require.Equal(t, allProviders, len(css)) for _, cs := range css { - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), numberOfProviders, numberOfProviders, false, false) require.NoError(t, err) } usedProviders := NewUsedProviders(nil) diff --git a/protocol/lavasession/end_to_end_lavasession_test.go b/protocol/lavasession/end_to_end_lavasession_test.go index 0b434855b6..c3fc59dc68 100644 --- a/protocol/lavasession/end_to_end_lavasession_test.go +++ b/protocol/lavasession/end_to_end_lavasession_test.go @@ -72,7 +72,7 @@ func TestHappyFlowE2EEmergency(t *testing.T) { err = psm.OnSessionDone(sps, cs.Session.RelayNum-skippedRelays) require.NoError(t, err) - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, maxCuForVirtualEpoch, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), 1, 1, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, maxCuForVirtualEpoch, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), 1, 1, false, false) require.NoError(t, err) } @@ -195,7 +195,7 @@ func prepareSessionsWithFirstRelay(t *testing.T, cuForFirstRequest uint64) (*Con require.NoError(t, err) // Consumer Side: - err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), 1, 1, false) + err = csm.OnSessionDone(cs.Session, servicedBlockNumber, cuForFirstRequest, time.Millisecond, cs.Session.CalculateExpectedLatency(2*time.Millisecond), (servicedBlockNumber - 1), 1, 1, false, false) require.NoError(t, err) require.Equal(t, cs.Session.CuSum, cuForFirstRequest) require.Equal(t, cs.Session.LatestRelayCu, latestRelayCuAfterDone) diff --git a/protocol/lavasession/single_consumer_session.go b/protocol/lavasession/single_consumer_session.go index 7aa0dc1304..3b757067e2 100644 --- a/protocol/lavasession/single_consumer_session.go +++ b/protocol/lavasession/single_consumer_session.go @@ -48,10 +48,12 @@ func (cs *SingleConsumerSession) getQosComputedResultOrZero() sdk.Dec { return sdk.ZeroDec() } -func (cs *SingleConsumerSession) CalculateQoS(latency, expectedLatency time.Duration, blockHeightDiff int64, numOfProviders int, servicersToCount int64) { +func (cs *SingleConsumerSession) CalculateQoS(latency, expectedLatency time.Duration, blockHeightDiff int64, numOfProviders int, servicersToCount int64, reduceAvailability bool) { // Add current Session QoS - cs.QoSInfo.TotalRelays++ // increase total relays - cs.QoSInfo.AnsweredRelays++ // increase answered relays + cs.QoSInfo.TotalRelays++ // increase total relays + if !reduceAvailability { // incase we want to reduce availability to this provider due to some reason we skip answered. + cs.QoSInfo.AnsweredRelays++ // increase answered relays + } if cs.QoSInfo.LastQoSReport == nil { cs.QoSInfo.LastQoSReport = &pairingtypes.QualityOfServiceReport{} diff --git a/protocol/rpcconsumer/consumer_relay_state_machine.go b/protocol/rpcconsumer/consumer_relay_state_machine.go index 3487572451..fdd99062b6 100644 --- a/protocol/rpcconsumer/consumer_relay_state_machine.go +++ b/protocol/rpcconsumer/consumer_relay_state_machine.go @@ -5,6 +5,7 @@ import ( "time" "github.com/lavanet/lava/v3/protocol/chainlib" + "github.com/lavanet/lava/v3/protocol/chainlib/extensionslib" common "github.com/lavanet/lava/v3/protocol/common" lavasession "github.com/lavanet/lava/v3/protocol/lavasession" "github.com/lavanet/lava/v3/protocol/metrics" @@ -26,6 +27,7 @@ type ConsumerRelaySender interface { sendRelayToProvider(ctx context.Context, protocolMessage chainlib.ProtocolMessage, relayProcessor *RelayProcessor, analytics *metrics.RelayMetrics) (errRet error) getProcessingTimeout(chainMessage chainlib.ChainMessage) (processingTimeout time.Duration, relayTimeout time.Duration) GetChainIdAndApiInterface() (string, string) + GetExtensionParser() *extensionslib.ExtensionParser } type tickerMetricSetterInf interface { @@ -117,7 +119,8 @@ func (crsm *ConsumerRelayStateMachine) GetRelayTaskChannel() chan RelayStateSend batchNumber := 0 // Set batch number // A channel to be notified processing was done, true means we have results and can return gotResults := make(chan bool, 1) - processingTimeout, relayTimeout := crsm.relaySender.getProcessingTimeout(crsm.GetProtocolMessage()) + protocolMessage := crsm.GetProtocolMessage() + processingTimeout, relayTimeout := crsm.relaySender.getProcessingTimeout(protocolMessage) if crsm.debugRelays { utils.LavaFormatDebug("Relay initiated with the following timeout schedule", utils.LogAttr("processingTimeout", processingTimeout), utils.LogAttr("newRelayTimeout", relayTimeout)) } @@ -125,6 +128,20 @@ func (crsm *ConsumerRelayStateMachine) GetRelayTaskChannel() chan RelayStateSend processingCtx, processingCtxCancel := context.WithTimeout(crsm.ctx, processingTimeout) defer processingCtxCancel() + apiName := protocolMessage.GetApi().Name + resetUsedOnce := true + setArchiveOnSpecialApi := func() { + if apiName == "tx" || apiName == "chunk" { + archiveExtensionArray := []string{"archive"} + protocolMessage.OverrideExtensions(archiveExtensionArray, crsm.relaySender.GetExtensionParser()) + protocolMessage.RelayPrivateData().Extensions = archiveExtensionArray + if resetUsedOnce { + resetUsedOnce = false + crsm.usedProviders = lavasession.NewUsedProviders(protocolMessage) + } + } + } + readResultsFromProcessor := func() { // ProcessResults is reading responses while blocking until the conditions are met utils.LavaFormatTrace("[StateMachine] Waiting for results", utils.LogAttr("batch", batchNumber)) @@ -133,6 +150,7 @@ func (crsm *ConsumerRelayStateMachine) GetRelayTaskChannel() chan RelayStateSend if crsm.parentRelayProcessor.HasRequiredNodeResults() { gotResults <- true } else { + setArchiveOnSpecialApi() gotResults <- false } } @@ -214,6 +232,7 @@ func (crsm *ConsumerRelayStateMachine) GetRelayTaskChannel() chan RelayStateSend case <-startNewBatchTicker.C: // Only trigger another batch for non BestResult relays or if we didn't pass the retry limit. if crsm.ShouldRetry(batchNumber) { + setArchiveOnSpecialApi() utils.LavaFormatTrace("[StateMachine] ticker triggered", utils.LogAttr("batch", batchNumber)) relayTaskChannel <- RelayStateSendInstructions{protocolMessage: crsm.GetProtocolMessage()} // Add ticker launch metrics diff --git a/protocol/rpcconsumer/consumer_relay_state_machine_test.go b/protocol/rpcconsumer/consumer_relay_state_machine_test.go index c42d003714..2b9ac5036f 100644 --- a/protocol/rpcconsumer/consumer_relay_state_machine_test.go +++ b/protocol/rpcconsumer/consumer_relay_state_machine_test.go @@ -20,6 +20,10 @@ type ConsumerRelaySenderMock struct { tickerValue time.Duration } +func (crsm *ConsumerRelaySenderMock) GetExtensionParser() *extensionslib.ExtensionParser { + return nil +} + func (crsm *ConsumerRelaySenderMock) sendRelayToProvider(ctx context.Context, protocolMessage chainlib.ProtocolMessage, relayProcessor *RelayProcessor, analytics *metrics.RelayMetrics) (errRet error) { return crsm.retValue } diff --git a/protocol/rpcconsumer/rpcconsumer_server.go b/protocol/rpcconsumer/rpcconsumer_server.go index d74a4fd141..2415604d3d 100644 --- a/protocol/rpcconsumer/rpcconsumer_server.go +++ b/protocol/rpcconsumer/rpcconsumer_server.go @@ -469,6 +469,10 @@ func (rpccs *RPCConsumerServer) CancelSubscriptionContext(subscriptionKey string } } +func (rpccs *RPCConsumerServer) GetExtensionParser() *extensionslib.ExtensionParser { + return rpccs.chainParser.ExtensionsParser() +} + func (rpccs *RPCConsumerServer) sendRelayToProvider( ctx context.Context, protocolMessage chainlib.ProtocolMessage, @@ -756,8 +760,13 @@ func (rpccs *RPCConsumerServer) sendRelayToProvider( ) } - errResponse = rpccs.consumerSessionManager.OnSessionDone(singleConsumerSession, latestBlock, chainlib.GetComputeUnits(protocolMessage), relayLatency, singleConsumerSession.CalculateExpectedLatency(expectedRelayTimeoutForQOS), expectedBH, numOfProviders, pairingAddressesLen, protocolMessage.GetApi().Category.HangingApi) // session done successfully isNodeError, _ := protocolMessage.CheckResponseError(localRelayResult.Reply.Data, localRelayResult.StatusCode) + reduceAvailability := false + if isNodeError { + // validate nodeError is matching our expectations for reducing availability. + reduceAvailability = strings.Contains(string(localRelayResult.Reply.Data), "The node does not track the shard ID") + } + errResponse = rpccs.consumerSessionManager.OnSessionDone(singleConsumerSession, latestBlock, chainlib.GetComputeUnits(protocolMessage), relayLatency, singleConsumerSession.CalculateExpectedLatency(expectedRelayTimeoutForQOS), expectedBH, numOfProviders, pairingAddressesLen, protocolMessage.GetApi().Category.HangingApi, reduceAvailability) // session done successfully localRelayResult.IsNodeError = isNodeError if rpccs.cache.CacheActive() && rpcclient.ValidateStatusCodes(localRelayResult.StatusCode, true) == nil { // in case the error is a node error we don't want to cache From f908cb52ea43fc5711f608876d2a7596ef10c591 Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Tue, 26 Nov 2024 17:43:50 +0100 Subject: [PATCH 2/3] feat: increase target version to 4.1.3 --- x/protocol/types/params.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/protocol/types/params.go b/x/protocol/types/params.go index 27ce27395b..d923b38f5f 100644 --- a/x/protocol/types/params.go +++ b/x/protocol/types/params.go @@ -12,7 +12,7 @@ import ( var _ paramtypes.ParamSet = (*Params)(nil) const ( - TARGET_VERSION = "4.1.2" + TARGET_VERSION = "4.1.3" MIN_VERSION = "3.1.0" ) From b0b3a9eab1c5014278a9997f89e66378134e3b18 Mon Sep 17 00:00:00 2001 From: Ran Mishael Date: Tue, 26 Nov 2024 17:55:09 +0100 Subject: [PATCH 3/3] adding experimental feature. --- protocol/rpcconsumer/rpcconsumer_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol/rpcconsumer/rpcconsumer_server.go b/protocol/rpcconsumer/rpcconsumer_server.go index 9a39ae8228..553cefab49 100644 --- a/protocol/rpcconsumer/rpcconsumer_server.go +++ b/protocol/rpcconsumer/rpcconsumer_server.go @@ -874,7 +874,7 @@ func (rpccs *RPCConsumerServer) sendRelayToProvider( isNodeError, _ := protocolMessage.CheckResponseError(localRelayResult.Reply.Data, localRelayResult.StatusCode) reduceAvailability := false - if isNodeError { + if isNodeError && (chainId == "NEAR" || chainId == "NEART") { // validate nodeError is matching our expectations for reducing availability. reduceAvailability = strings.Contains(string(localRelayResult.Reply.Data), "The node does not track the shard ID") }