Stream fallback: error handler (#12173)

* Stop retry flow for error handling (log trigger) (#12026) * return timeout for retry interval * identify retryTimeout + placeholder for err handler * err codes * comments * added function and tests for handling err code * add checkErrorHandler (#12037) * Stop retry flow for error handling (conditional trigger) (#12032) * err codes for conditionals * unit tests for conditional * refactor streams error handler * fix mercury v0.2 request handling * fix mercury 0.3 DoRequest * fix tests * connect check error callback * add todo * add todo * improve comments * polish 0.2 * fix debug.go, refactor eth_call on checkCallback and checkErrorHandler, fix some minor comments (typo and test) * small fixes * calculate retry config for conditionals * rename to clarify function * cleanup pipeline execution errors * fix unit tests for v02_request.go * Fix bug in 0.3 request error code * add state assertion to single feed request tests * add test case for invalid response bytes * add tests for more retryable errors * add tests for retryable -> error conversion * polish 0.2 combining multiple feeds * add more tests for different combination of feed responses * remove unused fields * unit tests for v03 * fix mercury_test.go * minor polishing * thread control: added function that accept a context * use threadCtrl.GoCtx to ensure the timeout is applied with the provided context * added timeout for mercury requests (including retries) * fix lint * set timeout to 10s * use GoCtx within request clients * lint * add more 0.3 tests * set err code to timeout if ctx is done * Finalize stream error codes, polish requests to return consistent nil bytes upon error, use HttpToStreamsErrCode everywhere * add tests for checkErrorHanlder * treat timeout as non retryable error codes * allow empty feed request which returns error code * update test contract with a flag for checkErr result * generate wrappers for LogTriggeredStreamsLookup * handling empty performData case for err handler * test (wip) * fixing contract * waiting for err handler logs (wip) * update contract and generate wrappers * lint * use startBlock instead of 1 * add missing arg * check multiple responses: - server timeout - unauthorized - bad req - internal server err - not found * fix test * cleanup * set timeout in http client * Add timeout test for streams (#12170) * Add timeout test * fix DoRequest() to consider ctx (client v0.3) * make sure we timeout in time * align thread control test * hacky context timeout * trying to wait for timeout with child context * push hack * update * another fix * fix * use ctx background * add log * Use new ctx to implement timeout * fix test * add v0.2 test --------- Co-authored-by: amirylm <[email protected]> * Empty Commit --------- Co-authored-by: Amir Y <[email protected]> Co-authored-by: Lei <[email protected]>
smartcontractkit · Feb 27, 2024 · 9659252 · 9659252
1 parent b649cf6
commit 9659252
Show file tree

Hide file tree

Showing 31 changed files with 2,179 additions and 326 deletions.
diff --git a/contracts/src/v0.8/automation/dev/MercuryRegistry.sol b/contracts/src/v0.8/automation/dev/MercuryRegistry.sol
@@ -144,6 +144,14 @@ contract MercuryRegistry is ConfirmedOwner, AutomationCompatibleInterface, Strea
     return (filteredValues.length > 0, performData);
   }
 
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view override returns (bool upkeepNeeded, bytes memory performData) {
+    // dummy function with default values
+    return (false, new bytes(0));
+  }
+
   // Use deviated off-chain values to update on-chain state.
   function performUpkeep(bytes calldata performData) external override {
     (bytes[] memory values /* bytes memory lookupData */, ) = abi.decode(performData, (bytes[], bytes));

diff --git a/contracts/src/v0.8/automation/dev/MercuryRegistryBatchUpkeep.sol b/contracts/src/v0.8/automation/dev/MercuryRegistryBatchUpkeep.sol
@@ -61,6 +61,14 @@ contract MercuryRegistryBatchUpkeep is ConfirmedOwner, AutomationCompatibleInter
     return i_registry.checkCallback(values, lookupData);
   }
 
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view override returns (bool upkeepNeeded, bytes memory performData) {
+    // dummy function with default values
+    return (false, new bytes(0));
+  }
+
   // Use the master registry to update state.
   function performUpkeep(bytes calldata performData) external override {
     i_registry.performUpkeep(performData);

diff --git a/contracts/src/v0.8/automation/interfaces/StreamsLookupCompatibleInterface.sol b/contracts/src/v0.8/automation/interfaces/StreamsLookupCompatibleInterface.sol
@@ -17,4 +17,17 @@ interface StreamsLookupCompatibleInterface {
     bytes[] memory values,
     bytes memory extraData
   ) external view returns (bool upkeepNeeded, bytes memory performData);
+
+  /**
+   * @notice this is a new, optional function in streams lookup. It is meant to surface streams lookup errors.
+   * @param errCode an uint value that represents the streams lookup error code.
+   * @param extraData context data from streams lookup process.
+   * @return upkeepNeeded boolean to indicate whether the keeper should call performUpkeep or not.
+   * @return performData bytes that the keeper should call performUpkeep with, if
+   * upkeep is needed. If you would like to encode data to decode later, try `abi.encode`.
+   */
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view returns (bool upkeepNeeded, bytes memory performData);
 }
diff --git a/contracts/src/v0.8/automation/testhelpers/LogTriggeredStreamsLookup.sol b/contracts/src/v0.8/automation/testhelpers/LogTriggeredStreamsLookup.sol
@@ -26,6 +26,7 @@ contract LogTriggeredStreamsLookup is ILogAutomation, StreamsLookupCompatibleInt
     bytes verified
   );
   event LimitOrderExecuted(uint256 indexed orderId, uint256 indexed amount, address indexed exchange); // keccak(LimitOrderExecuted(uint256,uint256,address)) => 0xd1ffe9e45581c11d7d9f2ed5f75217cd4be9f8b7eee6af0f6d03f46de53956cd
+  event IgnoringErrorHandlerData();
 
   ArbSys internal constant ARB_SYS = ArbSys(0x0000000000000000000000000000000000000064);
   IVerifierProxy internal constant VERIFIER = IVerifierProxy(0x09DFf56A4fF44e0f4436260A04F5CFa65636A481);
@@ -42,10 +43,12 @@ contract LogTriggeredStreamsLookup is ILogAutomation, StreamsLookupCompatibleInt
   string public feedParamKey = "feedIdHex";
   string public timeParamKey = "blockNumber";
   uint256 public counter;
+  bool public checkErrReturnBool;
 
-  constructor(bool _useArbitrumBlockNum, bool _verify) {
+  constructor(bool _useArbitrumBlockNum, bool _verify, bool _checkErrReturnBool) {
     useArbitrumBlockNum = _useArbitrumBlockNum;
     verify = _verify;
+    checkErrReturnBool = _checkErrReturnBool;
     counter = 0;
   }
 
@@ -94,6 +97,10 @@ contract LogTriggeredStreamsLookup is ILogAutomation, StreamsLookupCompatibleInt
   }
 
   function performUpkeep(bytes calldata performData) external override {
+    if (performData.length == 0) {
+      emit IgnoringErrorHandlerData();
+      return;
+    }
     (bytes[] memory values, bytes memory extraData) = abi.decode(performData, (bytes[], bytes));
     (uint256 orderId, uint256 amount, address exchange, bytes32 logTopic0) = abi.decode(
       extraData,
@@ -130,6 +137,14 @@ contract LogTriggeredStreamsLookup is ILogAutomation, StreamsLookupCompatibleInt
     return (true, performData);
   }
 
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view override returns (bool upkeepNeeded, bytes memory performData) {
+    // dummy function with default values
+    return (checkErrReturnBool, new bytes(0));
+  }
+
   function getBlockNumber() internal view returns (uint256) {
     if (useArbitrumBlockNum) {
       return ARB_SYS.arbBlockNumber();

diff --git a/contracts/src/v0.8/tests/StreamsLookupUpkeep.sol b/contracts/src/v0.8/tests/StreamsLookupUpkeep.sol
@@ -88,6 +88,14 @@ contract StreamsLookupUpkeep is AutomationCompatibleInterface, StreamsLookupComp
     return (callbackReturnBool, performData);
   }
 
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view override returns (bool upkeepNeeded, bytes memory performData) {
+    // dummy function with default values
+    return (false, new bytes(0));
+  }
+
   function checkUpkeep(bytes calldata data) external view returns (bool, bytes memory) {
     if (!eligible()) {
       return (false, data);

diff --git a/contracts/src/v0.8/tests/VerifiableLoadLogTriggerUpkeep.sol b/contracts/src/v0.8/tests/VerifiableLoadLogTriggerUpkeep.sol
@@ -116,4 +116,12 @@ contract VerifiableLoadLogTriggerUpkeep is VerifiableLoadBase, StreamsLookupComp
     bytes memory performData = abi.encode(values, extraData);
     return (true, performData);
   }
+
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view override returns (bool upkeepNeeded, bytes memory performData) {
+    // dummy function with default values
+    return (false, new bytes(0));
+  }
 }
diff --git a/contracts/src/v0.8/tests/VerifiableLoadStreamsLookupUpkeep.sol b/contracts/src/v0.8/tests/VerifiableLoadStreamsLookupUpkeep.sol
@@ -16,6 +16,14 @@ contract VerifiableLoadStreamsLookupUpkeep is VerifiableLoadBase, StreamsLookupC
     return (true, performData);
   }
 
+  function checkErrorHandler(
+    uint256 errCode,
+    bytes memory extraData
+  ) external view override returns (bool upkeepNeeded, bytes memory performData) {
+    // dummy function with default values
+    return (false, new bytes(0));
+  }
+
   function checkUpkeep(bytes calldata checkData) external returns (bool, bytes memory) {
     uint256 startGas = gasleft();
     uint256 upkeepId = abi.decode(checkData, (uint256));

diff --git a/...rs/generated/log_triggered_streams_lookup_wrapper/log_triggered_streams_lookup_wrapper.go b/...rs/generated/log_triggered_streams_lookup_wrapper/log_triggered_streams_lookup_wrapper.go
diff --git a/...pers/generated/streams_lookup_compatible_interface/streams_lookup_compatible_interface.go b/...pers/generated/streams_lookup_compatible_interface/streams_lookup_compatible_interface.go
diff --git a/core/gethwrappers/generated/streams_lookup_upkeep_wrapper/streams_lookup_upkeep_wrapper.go b/core/gethwrappers/generated/streams_lookup_upkeep_wrapper/streams_lookup_upkeep_wrapper.go
diff --git a/.../verifiable_load_log_trigger_upkeep_wrapper/verifiable_load_log_trigger_upkeep_wrapper.go b/.../verifiable_load_log_trigger_upkeep_wrapper/verifiable_load_log_trigger_upkeep_wrapper.go
diff --git a/...iable_load_streams_lookup_upkeep_wrapper/verifiable_load_streams_lookup_upkeep_wrapper.go b/...iable_load_streams_lookup_upkeep_wrapper/verifiable_load_streams_lookup_upkeep_wrapper.go
diff --git a/core/gethwrappers/generation/generated-wrapper-dependency-versions-do-not-edit.txt b/core/gethwrappers/generation/generated-wrapper-dependency-versions-do-not-edit.txt
@@ -46,7 +46,7 @@ keeper_registry_wrapper_2_1: ../../contracts/solc/v0.8.16/KeeperRegistry2_1/Keep
 keeper_registry_wrapper_2_2: ../../contracts/solc/v0.8.16/AutomationRegistry2_2/AutomationRegistry2_2.abi ../../contracts/solc/v0.8.16/AutomationRegistry2_2/AutomationRegistry2_2.bin 8f1c6d3eddf36201a556a101350abd6eb733d4c3177da2ddb5eca0d24d9a88ef
 keepers_vrf_consumer: ../../contracts/solc/v0.8.6/KeepersVRFConsumer/KeepersVRFConsumer.abi ../../contracts/solc/v0.8.6/KeepersVRFConsumer/KeepersVRFConsumer.bin fa75572e689c9e84705c63e8dbe1b7b8aa1a8fe82d66356c4873d024bb9166e8
 log_emitter: ../../contracts/solc/v0.8.19/LogEmitter/LogEmitter.abi ../../contracts/solc/v0.8.19/LogEmitter/LogEmitter.bin 4b129ab93432c95ff9143f0631323e189887668889e0b36ccccf18a571e41ccf
-log_triggered_streams_lookup_wrapper: ../../contracts/solc/v0.8.16/LogTriggeredStreamsLookup/LogTriggeredStreamsLookup.abi ../../contracts/solc/v0.8.16/LogTriggeredStreamsLookup/LogTriggeredStreamsLookup.bin f8da43a927c1a66238a9f4fd5d5dd7e280e361daa0444da1f7f79498ace901e1
+log_triggered_streams_lookup_wrapper: ../../contracts/solc/v0.8.16/LogTriggeredStreamsLookup/LogTriggeredStreamsLookup.abi ../../contracts/solc/v0.8.16/LogTriggeredStreamsLookup/LogTriggeredStreamsLookup.bin 920fff3b662909f12ed11b47d168036ffa74ad52070a94e2fa26cdad5e428b4e
 log_upkeep_counter_wrapper: ../../contracts/solc/v0.8.6/LogUpkeepCounter/LogUpkeepCounter.abi ../../contracts/solc/v0.8.6/LogUpkeepCounter/LogUpkeepCounter.bin 42426bbb83f96dfbe55fc576d6c65020eaeed690e2289cf99b0c4aa810a5f4ec
 mock_aggregator_proxy: ../../contracts/solc/v0.8.6/MockAggregatorProxy/MockAggregatorProxy.abi ../../contracts/solc/v0.8.6/MockAggregatorProxy/MockAggregatorProxy.bin b16c108f3dd384c342ddff5e94da7c0a8d39d1be5e3d8f2cf61ecc7f0e50ff42
 mock_ethlink_aggregator_wrapper: ../../contracts/solc/v0.6/MockETHLINKAggregator/MockETHLINKAggregator.abi ../../contracts/solc/v0.6/MockETHLINKAggregator/MockETHLINKAggregator.bin 1c52c24f797b8482aa12b8251dcea1c072827bd5b3426b822621261944b99ca0
@@ -66,16 +66,16 @@ solidity_vrf_request_id_v08: ../../contracts/solc/v0.8.6/VRFRequestIDBaseTestHel
 solidity_vrf_v08_verifier_wrapper: ../../contracts/solc/v0.8.6/VRFTestHelper/VRFTestHelper.abi ../../contracts/solc/v0.8.6/VRFTestHelper/VRFTestHelper.bin f37f8b21a81c113085c6137835a2246db6ebda07da455c4f2b5c7ec60c725c3b
 solidity_vrf_verifier_wrapper: ../../contracts/solc/v0.6/VRFTestHelper/VRFTestHelper.abi ../../contracts/solc/v0.6/VRFTestHelper/VRFTestHelper.bin 44c2b67d8d2990ab580453deb29d63508c6147a3dc49908a1db563bef06e6474
 solidity_vrf_wrapper: ../../contracts/solc/v0.6/VRF/VRF.abi ../../contracts/solc/v0.6/VRF/VRF.bin 04ede5b83c06ba5b76ef99c081c72928007d8a7aaefcf21449a46a07cbd4bfc2
-streams_lookup_compatible_interface: ../../contracts/solc/v0.8.16/StreamsLookupCompatibleInterface/StreamsLookupCompatibleInterface.abi ../../contracts/solc/v0.8.16/StreamsLookupCompatibleInterface/StreamsLookupCompatibleInterface.bin feb92cc666df21ea04ab9d7a588a513847b01b2f66fc167d06ab28ef2b17e015
-streams_lookup_upkeep_wrapper: ../../contracts/solc/v0.8.16/StreamsLookupUpkeep/StreamsLookupUpkeep.abi ../../contracts/solc/v0.8.16/StreamsLookupUpkeep/StreamsLookupUpkeep.bin b1a598963cacac51ed4706538d0f142bdc0d94b9a4b13e2d402131cdf05c9bcf
+streams_lookup_compatible_interface: ../../contracts/solc/v0.8.16/StreamsLookupCompatibleInterface/StreamsLookupCompatibleInterface.abi ../../contracts/solc/v0.8.16/StreamsLookupCompatibleInterface/StreamsLookupCompatibleInterface.bin 2861f553fb4731e89126b13319462df674727005a51982d1e617e2c2e44fa422
+streams_lookup_upkeep_wrapper: ../../contracts/solc/v0.8.16/StreamsLookupUpkeep/StreamsLookupUpkeep.abi ../../contracts/solc/v0.8.16/StreamsLookupUpkeep/StreamsLookupUpkeep.bin 37e3a61091cc2a156539dd4aaff987e07577118aa02e97931a647df55705465e
 test_api_consumer_wrapper: ../../contracts/solc/v0.6/TestAPIConsumer/TestAPIConsumer.abi ../../contracts/solc/v0.6/TestAPIConsumer/TestAPIConsumer.bin ed10893cb18894c18e275302329c955f14ea2de37ee044f84aa1e067ac5ea71e
 trusted_blockhash_store: ../../contracts/solc/v0.8.6/TrustedBlockhashStore/TrustedBlockhashStore.abi ../../contracts/solc/v0.8.6/TrustedBlockhashStore/TrustedBlockhashStore.bin 98cb0dc06c15af5dcd3b53bdfc98e7ed2489edc96a42203294ac2fc0efdda02b
 type_and_version_interface_wrapper: ../../contracts/solc/v0.8.6/KeeperRegistry1_2/TypeAndVersionInterface.abi ../../contracts/solc/v0.8.6/KeeperRegistry1_2/TypeAndVersionInterface.bin bc9c3a6e73e3ebd5b58754df0deeb3b33f4bb404d5709bb904aed51d32f4b45e
 upkeep_counter_wrapper: ../../contracts/solc/v0.8.16/UpkeepCounter/UpkeepCounter.abi ../../contracts/solc/v0.8.16/UpkeepCounter/UpkeepCounter.bin 77f000229a501f638dd2dc439859257f632894c728b31e68aea4f6d6c52f1b71
 upkeep_perform_counter_restrictive_wrapper: ../../contracts/solc/v0.8.16/UpkeepPerformCounterRestrictive/UpkeepPerformCounterRestrictive.abi ../../contracts/solc/v0.8.16/UpkeepPerformCounterRestrictive/UpkeepPerformCounterRestrictive.bin 20955b21acceb58355fa287b29194a73edf5937067ba7140667301017cb2b24c
 upkeep_transcoder: ../../contracts/solc/v0.8.6/UpkeepTranscoder/UpkeepTranscoder.abi ../../contracts/solc/v0.8.6/UpkeepTranscoder/UpkeepTranscoder.bin 336c92a981597be26508455f81a908a0784a817b129a59686c5b2c4afcba730a
-verifiable_load_log_trigger_upkeep_wrapper: ../../contracts/solc/v0.8.16/VerifiableLoadLogTriggerUpkeep/VerifiableLoadLogTriggerUpkeep.abi ../../contracts/solc/v0.8.16/VerifiableLoadLogTriggerUpkeep/VerifiableLoadLogTriggerUpkeep.bin fb674ba44c0e8f3b385cd10b2f7dea5cd07b5f38df08066747e8b1542e152557
-verifiable_load_streams_lookup_upkeep_wrapper: ../../contracts/solc/v0.8.16/VerifiableLoadStreamsLookupUpkeep/VerifiableLoadStreamsLookupUpkeep.abi ../../contracts/solc/v0.8.16/VerifiableLoadStreamsLookupUpkeep/VerifiableLoadStreamsLookupUpkeep.bin 785f68c44bfff070505eaa65e38a1af94046e5f9afc1189bcf2c8cfcd1102d66
+verifiable_load_log_trigger_upkeep_wrapper: ../../contracts/solc/v0.8.16/VerifiableLoadLogTriggerUpkeep/VerifiableLoadLogTriggerUpkeep.abi ../../contracts/solc/v0.8.16/VerifiableLoadLogTriggerUpkeep/VerifiableLoadLogTriggerUpkeep.bin ee5c608e4e84c80934e42b0c02a49624840adf10b50c91f688bf8f0c7c6994c2
+verifiable_load_streams_lookup_upkeep_wrapper: ../../contracts/solc/v0.8.16/VerifiableLoadStreamsLookupUpkeep/VerifiableLoadStreamsLookupUpkeep.abi ../../contracts/solc/v0.8.16/VerifiableLoadStreamsLookupUpkeep/VerifiableLoadStreamsLookupUpkeep.bin 58f1f6b31a313e04ceb3e0e0f0393bc195cc2f4784a3b0e14a80a86fc836f427
 verifiable_load_upkeep_wrapper: ../../contracts/solc/v0.8.16/VerifiableLoadUpkeep/VerifiableLoadUpkeep.abi ../../contracts/solc/v0.8.16/VerifiableLoadUpkeep/VerifiableLoadUpkeep.bin a3e02c43756ea91e7ce4b81e48c11648f1d12f6663c236780147e41dfa36ebee
 vrf_consumer_v2: ../../contracts/solc/v0.8.6/VRFConsumerV2/VRFConsumerV2.abi ../../contracts/solc/v0.8.6/VRFConsumerV2/VRFConsumerV2.bin 9ef258bf8e9f8d880fd229ceb145593d91e24fc89366baa0bf19169c5787d15f
 vrf_consumer_v2_plus_upgradeable_example: ../../contracts/solc/v0.8.6/VRFConsumerV2PlusUpgradeableExample/VRFConsumerV2PlusUpgradeableExample.abi ../../contracts/solc/v0.8.6/VRFConsumerV2PlusUpgradeableExample/VRFConsumerV2PlusUpgradeableExample.bin 3155c611e4d6882e9324b6e975033b31356776ea8b031ca63d63da37589d583b

diff --git a/core/scripts/chaincli/handler/debug.go b/core/scripts/chaincli/handler/debug.go
@@ -324,16 +324,17 @@ func (k *Keeper) Debug(ctx context.Context, args []string) {
 			checkResults := []ocr2keepers.CheckResult{automationCheckResult}
 
 			var values [][]byte
-			values, err = streams.DoMercuryRequest(ctx, streamsLookup, checkResults, 0)
+			var errCode encoding.ErrCode
+			values, errCode, err = streams.DoMercuryRequest(ctx, streamsLookup, checkResults, 0)
 
 			if checkResults[0].IneligibilityReason == uint8(encoding.UpkeepFailureReasonInvalidRevertDataInput) {
 				resolveIneligible("upkeep used invalid revert data")
 			}
-			if checkResults[0].PipelineExecutionState == uint8(encoding.InvalidMercuryRequest) {
-				resolveIneligible("the data streams request data is invalid")
-			}
 			if err != nil {
-				failCheckConfig("failed to do data streams request ", err)
+				failCheckConfig("pipeline execution error, failed to do data streams request ", err)
+			}
+			if errCode != encoding.ErrCodeNil {
+				failCheckConfig(fmt.Sprintf("data streams error, failed to do data streams request with error code %d", errCode), nil)
 			}
 
 			// do checkCallback