From f499a16a7303cdd9bac5754ab09cfcac42349639 Mon Sep 17 00:00:00 2001 From: Sam Davies Date: Thu, 14 Sep 2023 13:52:10 -0400 Subject: [PATCH] Implement Data Streams plugin --- .../scripts/native_solc_compile_all_llo-feeds | 8 +- .../src/v0.8/llo-feeds/ConfigurationStore.sol | 63 +++ .../src/v0.8/llo-feeds/StreamsConfig.sol | 500 ++++++++++++++++++ .../llo-feeds/interfaces/IStreamsConfig.sol | 97 ++++ core/scripts/go.sum | 4 +- core/services/functions/connector_handler.go | 4 +- core/services/gateway/connector/connector.go | 4 +- core/services/job/orm.go | 2 +- core/services/job/spawner.go | 10 +- .../keystore/keys/ocr2key/cosmos_keyring.go | 59 ++- .../keystore/keys/ocr2key/evm_keyring.go | 49 +- .../keys/ocr2key/generic_key_bundle.go | 9 + .../keystore/keys/ocr2key/key_bundle.go | 8 + .../keystore/keys/ocr2key/solana_keyring.go | 46 +- .../keystore/keys/starkkey/ocr2key.go | 9 +- core/services/ocr2/delegate.go | 94 +++- .../evmregistry/v21/logprovider/recoverer.go | 2 +- core/services/ocr2/plugins/s4/plugin_test.go | 2 +- .../ocr2/plugins/streams/config/config.go | 52 ++ .../ocr2/plugins/streams/helpers_test.go | 463 ++++++++++++++++ .../ocr2/plugins/streams/integration_test.go | 363 +++++++++++++ core/services/ocr2/plugins/streams/plugin.go | 1 + core/services/ocr2/validate/validate.go | 12 + core/services/relay/evm/evm.go | 50 +- .../relay/evm/functions/logpoller_wrapper.go | 4 +- .../services/relay/evm/mercury/transmitter.go | 6 +- .../relay/evm/mercury/transmitter_test.go | 33 +- .../relay/evm/mercury/v1/data_source.go | 2 +- .../relay/evm/mocks/loop_relay_adapter.go | 26 + core/services/relay/evm/streams_provider.go | 86 +++ core/services/streams/data_source.go | 91 ++++ core/services/streams/data_source_test.go | 90 ++++ core/services/streams/delegate.go | 90 ++++ core/services/streams/keyring.go | 60 +++ core/services/streams/keyring_test.go | 7 + core/services/streams/orm.go | 44 ++ core/services/streams/orm_test.go | 71 +++ core/services/streams/stream.go | 118 +++++ core/services/streams/stream_cache.go | 43 ++ core/services/streams/stream_cache_test.go | 73 +++ core/services/streams/transmitter.go | 74 +++ core/services/vrf/v1/listener_v1.go | 2 +- .../migrations/0213_create_streams.sql | 10 + go.mod | 11 + go.sum | 15 +- integration-tests/go.sum | 7 + 46 files changed, 2781 insertions(+), 93 deletions(-) create mode 100644 contracts/src/v0.8/llo-feeds/ConfigurationStore.sol create mode 100644 contracts/src/v0.8/llo-feeds/StreamsConfig.sol create mode 100644 contracts/src/v0.8/llo-feeds/interfaces/IStreamsConfig.sol create mode 100644 core/services/ocr2/plugins/streams/config/config.go create mode 100644 core/services/ocr2/plugins/streams/helpers_test.go create mode 100644 core/services/ocr2/plugins/streams/integration_test.go create mode 100644 core/services/ocr2/plugins/streams/plugin.go create mode 100644 core/services/relay/evm/streams_provider.go create mode 100644 core/services/streams/data_source.go create mode 100644 core/services/streams/data_source_test.go create mode 100644 core/services/streams/delegate.go create mode 100644 core/services/streams/keyring.go create mode 100644 core/services/streams/keyring_test.go create mode 100644 core/services/streams/orm.go create mode 100644 core/services/streams/orm_test.go create mode 100644 core/services/streams/stream.go create mode 100644 core/services/streams/stream_cache.go create mode 100644 core/services/streams/stream_cache_test.go create mode 100644 core/services/streams/transmitter.go create mode 100644 core/store/migrate/migrations/0213_create_streams.sql diff --git a/contracts/scripts/native_solc_compile_all_llo-feeds b/contracts/scripts/native_solc_compile_all_llo-feeds index 2caa6fb98de..59386d89439 100755 --- a/contracts/scripts/native_solc_compile_all_llo-feeds +++ b/contracts/scripts/native_solc_compile_all_llo-feeds @@ -33,6 +33,10 @@ compileContract llo-feeds/VerifierProxy.sol compileContract llo-feeds/FeeManager.sol compileContract llo-feeds/RewardManager.sol -#Test | Mocks +# Test | Mocks compileContract llo-feeds/test/mocks/ErroredVerifier.sol -compileContract llo-feeds/test/mocks/ExposedVerifier.sol \ No newline at end of file +compileContract llo-feeds/test/mocks/ExposedVerifier.sol + +# Streams +compileContract llo-feeds/ConfigurationStore.sol +compileContract llo-feeds/StreamsConfig.sol diff --git a/contracts/src/v0.8/llo-feeds/ConfigurationStore.sol b/contracts/src/v0.8/llo-feeds/ConfigurationStore.sol new file mode 100644 index 00000000000..8eaa03f76c7 --- /dev/null +++ b/contracts/src/v0.8/llo-feeds/ConfigurationStore.sol @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: UNLICENSED +pragma solidity ^0.8.13; + +struct ChannelDefinition { + // e.g. evm, solana, CosmWasm, kalechain, etc... + string reportFormat; + // Specifies the chain on which this channel can be verified. Currently uses + // CCIP chain selectors, but lots of other schemes are possible as well. + uint64 chainSelector; + // We assume that StreamIDs is always non-empty and that the 0-th stream + // contains the verification price in LINK and the 1-st stream contains the + // verification price in the native coin. + string[] streamIDs; +} + +contract ConfigurationStore { + //////////////////////// + // protocol instance management + //////////////////////// + + ChannelDefinition[] private s_channelDefinitions; + + // setProductionConfig() onlyOwner -- the usual OCR way + // sets config for the production protocol instance + + // setStagingConfig() onlyOwner -- the usual OCR way + // sets config for the staging protocol instance + + // promoteStagingConfig() onlyOwner + // this will trigger the following: + // - offchain ShouldRetireCache will start returning true for the old (production) + // protocol instance + // - once the old production instance retires it will generate a handover + // retirement report + // - the staging instance will become the new production instance once + // any honest oracle that is on both instances forward the retirement + // report from the old instance to the new instace via the + // PredecessorRetirementReportCache + // + // Note: the promotion flow only works if the previous production instance + // is working correctly & generating reports. If that's not the case, the + // owner is expected to "setProductionConfig" directly instead. This will + // cause "gaps" to be created, but that seems unavoidable in such a scenario. + + //////////////////////// + // channel management + //////////////////////// + + addChannel(ChannelDefinition) onlyOwner { + // TODO + } + + removeChannel(bytes32 channelId) onlyOwner { + // TODO + + } + + getChannelDefinitions() onlyEOA public view returns (ChannelDefinition[] memory) { + // TODO + + } + // used by ChannelDefinitionCache +} \ No newline at end of file diff --git a/contracts/src/v0.8/llo-feeds/StreamsConfig.sol b/contracts/src/v0.8/llo-feeds/StreamsConfig.sol new file mode 100644 index 00000000000..a4f9313354e --- /dev/null +++ b/contracts/src/v0.8/llo-feeds/StreamsConfig.sol @@ -0,0 +1,500 @@ +// SPDX-License-Identifier: MIT +pragma solidity 0.8.16; + +import {ConfirmedOwner} from "../shared/access/ConfirmedOwner.sol"; +import {IStreamsConfig} from "./interfaces/IStreamsConfig.sol"; +import {TypeAndVersionInterface} from "../interfaces/TypeAndVersionInterface.sol"; +import {IERC165} from "../vendor/openzeppelin-solidity/v4.8.3/contracts/interfaces/IERC165.sol"; +import {Common} from "./libraries/Common.sol"; + +// OCR2 standard +uint256 constant MAX_NUM_ORACLES = 31; + +/* + * The verifier contract is used to verify offchain reports signed + * by DONs. A report consists of a price, block number and feed Id. It + * represents the observed price of an asset at a specified block number for + * a feed. The verifier contract is used to verify that such reports have + * been signed by the correct signers. + **/ +contract StreamsConfig is IStreamsConfig, ConfirmedOwner, TypeAndVersionInterface { + // The first byte of the mask can be 0, because we only ever have 31 oracles + uint256 internal constant ORACLE_MASK = 0x0001010101010101010101010101010101010101010101010101010101010101; + + enum Role { + // Default role for an oracle address. This means that the oracle address + // is not a signer + Unset, + // Role given to an oracle address that is allowed to sign feed data + Signer + } + + struct Signer { + // Index of oracle in a configuration + uint8 index; + // The oracle's role + Role role; + } + + struct Config { + // Fault tolerance + uint8 f; + // Marks whether or not a configuration is active + bool isActive; + // Map of signer addresses to oracles + mapping(address => Signer) oracles; + } + + struct VerifierState { + // The number of times a new configuration + /// has been set + uint32 configCount; + // The block number of the block the last time + /// the configuration was updated. + uint32 latestConfigBlockNumber; + // The latest epoch a report was verified for + uint32 latestEpoch; + // Whether or not the verifier for this feed has been deactivated + bool isDeactivated; + /// The latest config digest set + bytes32 latestConfigDigest; + /// The previously set config + Config s_verificationDataConfig; + } + + /// @notice This event is emitted when a new report is verified. + /// It is used to keep a historical record of verified reports. + event ReportVerified(address requester); + + /// @notice This event is emitted whenever a new configuration is set. It triggers a new run of the offchain reporting protocol. + event ConfigSet( + uint32 previousConfigBlockNumber, + bytes32 configDigest, + uint64 configCount, + address[] signers, + bytes32[] offchainTransmitters, + uint8 f, + bytes onchainConfig, + uint64 offchainConfigVersion, + bytes offchainConfig + ); + + /// @notice This event is emitted whenever a configuration is deactivated + event ConfigDeactivated(bytes32 configDigest); + + /// @notice This event is emitted whenever a configuration is activated + event ConfigActivated(bytes32 configDigest); + + /// @notice This error is thrown whenever an address tries + /// to exeecute a transaction that it is not authorized to do so + error AccessForbidden(); + + /// @notice This error is thrown whenever a zero address is passed + error ZeroAddress(); + + /// @notice This error is thrown whenever the config digest + /// is empty + error DigestEmpty(); + + /// @notice This error is thrown whenever the config digest + /// passed in has not been set in this verifier + /// @param configDigest The config digest that has not been set + error DigestNotSet(bytes32 configDigest); + + /// @notice This error is thrown whenever the config digest + /// has been deactivated + /// @param configDigest The config digest that is inactive + error DigestInactive(bytes32 configDigest); + + /// @notice This error is thrown whenever trying to set a config + /// with a fault tolerance of 0 + error FaultToleranceMustBePositive(); + + /// @notice This error is thrown whenever a report is signed + /// with more than the max number of signers + /// @param numSigners The number of signers who have signed the report + /// @param maxSigners The maximum number of signers that can sign a report + error ExcessSigners(uint256 numSigners, uint256 maxSigners); + + /// @notice This error is thrown whenever a report is signed + /// with less than the minimum number of signers + /// @param numSigners The number of signers who have signed the report + /// @param minSigners The minimum number of signers that need to sign a report + error InsufficientSigners(uint256 numSigners, uint256 minSigners); + + /// @notice This error is thrown whenever a report is signed + /// with an incorrect number of signers + /// @param numSigners The number of signers who have signed the report + /// @param expectedNumSigners The expected number of signers that need to sign + /// a report + error IncorrectSignatureCount(uint256 numSigners, uint256 expectedNumSigners); + + /// @notice This error is thrown whenever the R and S signer components + /// have different lengths + /// @param rsLength The number of r signature components + /// @param ssLength The number of s signature components + error MismatchedSignatures(uint256 rsLength, uint256 ssLength); + + /// @notice This error is thrown whenever setting a config with duplicate signatures + error NonUniqueSignatures(); + + /// @notice This error is thrown whenever a report fails to verify due to bad or duplicate signatures + error BadVerification(); + + /// @notice This error is thrown whenever the admin tries to deactivate + /// the latest config digest + /// @param configDigest The latest config digest + error CannotDeactivateLatestConfig(bytes32 configDigest); + + /// @notice The address of the verifier proxy + address private immutable i_verifierProxyAddr; + + /// @notice Verifier state + VerifierState internal s_feedVerifierState; + + /// @param verifierProxyAddr The address of the VerifierProxy contract + constructor(address verifierProxyAddr) ConfirmedOwner(msg.sender) { + if (verifierProxyAddr == address(0)) revert ZeroAddress(); + i_verifierProxyAddr = verifierProxyAddr; + } + + modifier checkConfigValid(uint256 numSigners, uint256 f) { + if (f == 0) revert FaultToleranceMustBePositive(); + if (numSigners > MAX_NUM_ORACLES) revert ExcessSigners(numSigners, MAX_NUM_ORACLES); + if (numSigners <= 3 * f) revert InsufficientSigners(numSigners, 3 * f + 1); + _; + } + + /// @inheritdoc IERC165 + function supportsInterface(bytes4 interfaceId) external pure override returns (bool isVerifier) { + return interfaceId == this.verify.selector; + } + + /// @inheritdoc TypeAndVersionInterface + function typeAndVersion() external pure override returns (string memory) { + return "Verifier 1.2.0"; + } + + /// @inheritdoc IVerifier + function verify( + bytes calldata signedReport, + address sender + ) external override returns (bytes memory verifierResponse) { + if (msg.sender != i_verifierProxyAddr) revert AccessForbidden(); + ( + bytes32[3] memory reportContext, + bytes memory reportData, + bytes32[] memory rs, + bytes32[] memory ss, + bytes32 rawVs + ) = abi.decode(signedReport, (bytes32[3], bytes, bytes32[], bytes32[], bytes32)); + + VerifierState storage feedVerifierState = s_feedVerifierState; + + // If the feed has been deactivated, do not verify the report + if (feedVerifierState.isDeactivated) { + revert DigestInactive(); + } + + // reportContext consists of: + // reportContext[0]: ConfigDigest + // reportContext[1]: 27 byte padding, 4-byte epoch and 1-byte round + // reportContext[2]: ExtraHash + bytes32 configDigest = reportContext[0]; + Config storage s_config = feedVerifierState.s_verificationDataConfigs[configDigest]; + + _validateReport( configDigest, rs, ss, s_config); + _updateEpoch(reportContext, feedVerifierState); + + bytes32 hashedReport = keccak256(reportData); + + _verifySignatures(hashedReport, reportContext, rs, ss, rawVs, s_config); + emit ReportVerified( sender); + + return reportData; + } + + /// @notice Validates parameters of the report + /// @param configDigest Config digest from the report + /// @param rs R components from the report + /// @param ss S components from the report + /// @param config Config for the given feed ID keyed on the config digest + function _validateReport( + bytes32 configDigest, + bytes32[] memory rs, + bytes32[] memory ss, + Config storage config + ) private view { + uint8 expectedNumSignatures = config.f + 1; + + if (!config.isActive) revert DigestInactive( configDigest); + if (rs.length != expectedNumSignatures) revert IncorrectSignatureCount(rs.length, expectedNumSignatures); + if (rs.length != ss.length) revert MismatchedSignatures(rs.length, ss.length); + } + + /** + * @notice Conditionally update the epoch for a feed + * @param reportContext Report context containing the epoch and round + * @param feedVerifierState Feed verifier state to conditionally update + */ + function _updateEpoch(bytes32[3] memory reportContext, VerifierState storage feedVerifierState) private { + uint40 epochAndRound = uint40(uint256(reportContext[1])); + uint32 epoch = uint32(epochAndRound >> 8); + if (epoch > feedVerifierState.latestEpoch) { + feedVerifierState.latestEpoch = epoch; + } + } + + /// @notice Verifies that a report has been signed by the correct + /// signers and that enough signers have signed the reports. + /// @param hashedReport The keccak256 hash of the raw report's bytes + /// @param reportContext The context the report was signed in + /// @param rs ith element is the R components of the ith signature on report. Must have at most MAX_NUM_ORACLES entries + /// @param ss ith element is the S components of the ith signature on report. Must have at most MAX_NUM_ORACLES entries + /// @param rawVs ith element is the the V component of the ith signature + /// @param s_config The config digest the report was signed for + function _verifySignatures( + bytes32 hashedReport, + bytes32[3] memory reportContext, + bytes32[] memory rs, + bytes32[] memory ss, + bytes32 rawVs, + Config storage s_config + ) private view { + bytes32 h = keccak256(abi.encodePacked(hashedReport, reportContext)); + // i-th byte counts number of sigs made by i-th signer + uint256 signedCount; + + Signer memory o; + address signerAddress; + uint256 numSigners = rs.length; + for (uint256 i; i < numSigners; ++i) { + signerAddress = ecrecover(h, uint8(rawVs[i]) + 27, rs[i], ss[i]); + o = s_config.oracles[signerAddress]; + if (o.role != Role.Signer) revert BadVerification(); + unchecked { + signedCount += 1 << (8 * o.index); + } + } + + if (signedCount & ORACLE_MASK != signedCount) revert BadVerification(); + } + + /// @inheritdoc IVerifier + function setConfig( + address[] memory signers, + bytes32[] memory offchainTransmitters, + uint8 f, + bytes memory onchainConfig, + uint64 offchainConfigVersion, + bytes memory offchainConfig, + Common.AddressAndWeight[] memory recipientAddressesAndWeights + ) external override checkConfigValid(signers.length, f) onlyOwner { + _setConfig( + block.chainid, + address(this), + 0, // 0 defaults to feedConfig.configCount + 1 + signers, + offchainTransmitters, + f, + onchainConfig, + offchainConfigVersion, + offchainConfig, + recipientAddressesAndWeights + ); + } + + /// @inheritdoc IVerifier + function setConfigFromSource( + uint256 sourceChainId, + address sourceAddress, + uint32 newConfigCount, + address[] memory signers, + bytes32[] memory offchainTransmitters, + uint8 f, + bytes memory onchainConfig, + uint64 offchainConfigVersion, + bytes memory offchainConfig, + Common.AddressAndWeight[] memory recipientAddressesAndWeights + ) external override checkConfigValid(signers.length, f) onlyOwner { + _setConfig( + sourceChainId, + sourceAddress, + newConfigCount, + signers, + offchainTransmitters, + f, + onchainConfig, + offchainConfigVersion, + offchainConfig, + recipientAddressesAndWeights + ); + } + + /// @notice Sets config based on the given arguments + /// @param sourceChainId Chain ID of source config + /// @param sourceAddress Address of source config Verifier + /// @param newConfigCount Optional param to force the new config count + /// @param signers addresses with which oracles sign the reports + /// @param offchainTransmitters CSA key for the ith Oracle + /// @param f number of faulty oracles the system can tolerate + /// @param onchainConfig serialized configuration used by the contract (and possibly oracles) + /// @param offchainConfigVersion version number for offchainEncoding schema + /// @param offchainConfig serialized configuration used by the oracles exclusively and only passed through the contract + /// @param recipientAddressesAndWeights the addresses and weights of all the recipients to receive rewards + function _setConfig( + uint256 sourceChainId, + address sourceAddress, + uint32 newConfigCount, + address[] memory signers, + bytes32[] memory offchainTransmitters, + uint8 f, + bytes memory onchainConfig, + uint64 offchainConfigVersion, + bytes memory offchainConfig, + Common.AddressAndWeight[] memory recipientAddressesAndWeights + ) internal { + VerifierState storage feedVerifierState = s_feedVerifierState; + + // Increment the number of times a config has been set first + if (newConfigCount > 0) feedVerifierState.configCount = newConfigCount; + else feedVerifierState.configCount++; + + bytes32 configDigest = _configDigestFromConfigData( + sourceChainId, + sourceAddress, + feedVerifierState.configCount, + signers, + offchainTransmitters, + f, + onchainConfig, + offchainConfigVersion, + offchainConfig + ); + + feedVerifierState.s_verificationDataConfigs[configDigest].f = f; + feedVerifierState.s_verificationDataConfigs[configDigest].isActive = true; + for (uint8 i; i < signers.length; ++i) { + address signerAddr = signers[i]; + if (signerAddr == address(0)) revert ZeroAddress(); + + // All signer roles are unset by default for a new config digest. + // Here the contract checks to see if a signer's address has already + // been set to ensure that the group of signer addresses that will + // sign reports with the config digest are unique. + bool isSignerAlreadySet = feedVerifierState.s_verificationDataConfigs[configDigest].oracles[signerAddr].role != + Role.Unset; + if (isSignerAlreadySet) revert NonUniqueSignatures(); + feedVerifierState.s_verificationDataConfigs[configDigest].oracles[signerAddr] = Signer({ + role: Role.Signer, + index: i + }); + } + + IVerifierProxy(i_verifierProxyAddr).setVerifier( + feedVerifierState.latestConfigDigest, + configDigest, + recipientAddressesAndWeights + ); + + emit ConfigSet( + feedVerifierState.latestConfigBlockNumber, + configDigest, + feedVerifierState.configCount, + signers, + offchainTransmitters, + f, + onchainConfig, + offchainConfigVersion, + offchainConfig + ); + + feedVerifierState.latestEpoch = 0; + feedVerifierState.latestConfigBlockNumber = uint32(block.number); + feedVerifierState.latestConfigDigest = configDigest; + } + + /// @notice Generates the config digest from config data + /// @param sourceChainId Chain ID of source config + /// @param sourceAddress Address of source config Verifier + /// @param configCount ordinal number of this config setting among all config settings over the life of this contract + /// @param signers ith element is address ith oracle uses to sign a report + /// @param offchainTransmitters ith element is address ith oracle used to transmit reports (in this case used for flexible additional field, such as CSA pub keys) + /// @param f maximum number of faulty/dishonest oracles the protocol can tolerate while still working correctly + /// @param onchainConfig serialized configuration used by the contract (and possibly oracles) + /// @param offchainConfigVersion version of the serialization format used for "offchainConfig" parameter + /// @param offchainConfig serialized configuration used by the oracles exclusively and only passed through the contract + /// @dev This function is a modified version of the method from OCR2Abstract + function _configDigestFromConfigData( + uint256 sourceChainId, + address sourceAddress, + uint64 configCount, + address[] memory signers, + bytes32[] memory offchainTransmitters, + uint8 f, + bytes memory onchainConfig, + uint64 offchainConfigVersion, + bytes memory offchainConfig + ) internal pure returns (bytes32) { + uint256 h = uint256( + keccak256( + abi.encode( + sourceChainId, + sourceAddress, + configCount, + signers, + offchainTransmitters, + f, + onchainConfig, + offchainConfigVersion, + offchainConfig + ) + ) + ); + uint256 prefixMask = type(uint256).max << (256 - 16); // 0xFFFF00..00 + // 0x0006 corresponds to ConfigDigestPrefixMercuryV02 in libocr + uint256 prefix = 0x0006 << (256 - 16); // 0x000600..00 + return bytes32((prefix & prefixMask) | (h & ~prefixMask)); + } + + /// @inheritdoc IVerifier + function activateConfig(bytes32 configDigest) external onlyOwner { + VerifierState storage feedVerifierState = s_feedVerifierState; + + if (configDigest == bytes32("")) revert DigestEmpty(); + if (feedVerifierState.s_verificationDataConfigs[configDigest].f == 0) revert DigestNotSet(configDigest); + feedVerifierState.s_verificationDataConfigs[configDigest].isActive = true; + emit ConfigActivated(configDigest); + } + + /// @inheritdoc IVerifier + function deactivateConfig(bytes32 configDigest) external onlyOwner { + VerifierState storage feedVerifierState = s_feedVerifierState; + + if (configDigest == bytes32("")) revert DigestEmpty(); + if (feedVerifierState.s_verificationDataConfigs[configDigest].f == 0) revert DigestNotSet(configDigest); + if (configDigest == feedVerifierState.latestConfigDigest) { + revert CannotDeactivateLatestConfig(configDigest); + } + feedVerifierState.s_verificationDataConfigs[configDigest].isActive = false; + emit ConfigDeactivated(configDigest); + } + + /// @inheritdoc IVerifier + function latestConfigDigestAndEpoch( + ) external view override returns (bool scanLogs, bytes32 configDigest, uint32 epoch) { + VerifierState storage feedVerifierState = s_feedVerifierState; + return (false, feedVerifierState.latestConfigDigest, feedVerifierState.latestEpoch); + } + + /// @inheritdoc IVerifier + function latestConfigDetails( + ) external view override returns (uint32 configCount, uint32 blockNumber, bytes32 configDigest) { + VerifierState storage feedVerifierState = s_feedVerifierState; + return ( + feedVerifierState.configCount, + feedVerifierState.latestConfigBlockNumber, + feedVerifierState.latestConfigDigest + ); + } +} diff --git a/contracts/src/v0.8/llo-feeds/interfaces/IStreamsConfig.sol b/contracts/src/v0.8/llo-feeds/interfaces/IStreamsConfig.sol new file mode 100644 index 00000000000..9e0d2d956af --- /dev/null +++ b/contracts/src/v0.8/llo-feeds/interfaces/IStreamsConfig.sol @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: MIT +pragma solidity 0.8.16; + +import {IERC165} from "../../vendor/openzeppelin-solidity/v4.8.3/contracts/interfaces/IERC165.sol"; +import {Common} from "../libraries/Common.sol"; + +interface IStreamsConfig is IERC165 { + /** + * @notice Verifies that the data encoded has been signed + * correctly by routing to the correct verifier. + * @param signedReport The encoded data to be verified. + * @param sender The address that requested to verify the contract. + * This is only used for logging purposes. + * @dev Verification is typically only done through the proxy contract so + * we can't just use msg.sender to log the requester as the msg.sender + * contract will always be the proxy. + * @return verifierResponse The encoded verified response. + */ + function verify(bytes calldata signedReport, address sender) external returns (bytes memory verifierResponse); + + /** + * @notice sets offchain reporting protocol configuration incl. participating oracles + * @param signers addresses with which oracles sign the reports + * @param offchainTransmitters CSA key for the ith Oracle + * @param f number of faulty oracles the system can tolerate + * @param onchainConfig serialized configuration used by the contract (and possibly oracles) + * @param offchainConfigVersion version number for offchainEncoding schema + * @param offchainConfig serialized configuration used by the oracles exclusively and only passed through the contract + * @param recipientAddressesAndWeights the addresses and weights of all the recipients to receive rewards + */ + function setConfig( + address[] memory signers, + bytes32[] memory offchainTransmitters, + uint8 f, + bytes memory onchainConfig, + uint64 offchainConfigVersion, + bytes memory offchainConfig, + Common.AddressAndWeight[] memory recipientAddressesAndWeights + ) external; + + /** + * @notice identical to `setConfig` except with args for sourceChainId and sourceAddress + * @param sourceChainId Chain ID of source config + * @param sourceAddress Address of source config Verifier + * @param newConfigCount Param to force the new config count + * @param signers addresses with which oracles sign the reports + * @param offchainTransmitters CSA key for the ith Oracle + * @param f number of faulty oracles the system can tolerate + * @param onchainConfig serialized configuration used by the contract (and possibly oracles) + * @param offchainConfigVersion version number for offchainEncoding schema + * @param offchainConfig serialized configuration used by the oracles exclusively and only passed through the contract + * @param recipientAddressesAndWeights the addresses and weights of all the recipients to receive rewards + */ + function setConfigFromSource( + uint256 sourceChainId, + address sourceAddress, + uint32 newConfigCount, + address[] memory signers, + bytes32[] memory offchainTransmitters, + uint8 f, + bytes memory onchainConfig, + uint64 offchainConfigVersion, + bytes memory offchainConfig, + Common.AddressAndWeight[] memory recipientAddressesAndWeights + ) external; + + /** + * @notice Activates the configuration for a config digest + * @param configDigest The config digest to activate + * @dev This function can be called by the contract admin to activate a configuration. + */ + function activateConfig(bytes32 configDigest) external; + + /** + * @notice Deactivates the configuration for a config digest + * @param configDigest The config digest to deactivate + * @dev This function can be called by the contract admin to deactivate an incorrect configuration. + */ + function deactivateConfig(bytes32 configDigest) external; + + /** + * @notice returns the latest config digest and epoch for a feed + * @return scanLogs indicates whether to rely on the configDigest and epoch + * returned or whether to scan logs for the Transmitted event instead. + * @return configDigest + * @return epoch + */ + function latestConfigDigestAndEpoch() external view returns (bool scanLogs, bytes32 configDigest, uint32 epoch); + + /** + * @notice information about current offchain reporting protocol configuration + * @return configCount ordinal number of current config, out of all configs applied to this contract so far + * @return blockNumber block at which this config was set + * @return configDigest domain-separation tag for current config + */ + function latestConfigDetails() external view returns (uint32 configCount, uint32 blockNumber, bytes32 configDigest); +} diff --git a/core/scripts/go.sum b/core/scripts/go.sum index a8cf979a0e6..ebe56ce6683 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1302,8 +1302,8 @@ github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoM github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= -github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= -github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= +github.com/stretchr/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= +github.com/stretchr/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a h1:YuO+afVc3eqrjiCUizNCxI53bl/BnPiVwXqLzqYTqgU= github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a/go.mod h1:/sfW47zCZp9FrtGcWyo1VjbgDaodxX9ovZvgLb/MxaA= github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= diff --git a/core/services/functions/connector_handler.go b/core/services/functions/connector_handler.go index 5496bbdefc1..20ad5e9e546 100644 --- a/core/services/functions/connector_handler.go +++ b/core/services/functions/connector_handler.go @@ -244,7 +244,7 @@ func (h *functionsConnectorHandler) handleOffchainRequest(request *OffchainReque defer cancel() err := h.listener.HandleOffchainRequest(ctx, request) if err != nil { - h.lggr.Errorw("internal error while processing", "id", request.RequestId, "error", err) + h.lggr.Errorw("internal error while processing", "id", request.RequestId, "err", err) h.mu.Lock() defer h.mu.Unlock() state, ok := h.heartbeatRequests[RequestID(request.RequestId)] @@ -303,7 +303,7 @@ func (h *functionsConnectorHandler) cacheNewRequestLocked(requestId RequestID, r func (h *functionsConnectorHandler) sendResponseAndLog(ctx context.Context, gatewayId string, requestBody *api.MessageBody, payload any) { err := h.sendResponse(ctx, gatewayId, requestBody, payload) if err != nil { - h.lggr.Errorw("failed to send response to gateway", "id", gatewayId, "error", err) + h.lggr.Errorw("failed to send response to gateway", "id", gatewayId, "err", err) } else { h.lggr.Debugw("sent to gateway", "id", gatewayId, "messageId", requestBody.MessageId, "donId", requestBody.DonId, "method", requestBody.Method) } diff --git a/core/services/gateway/connector/connector.go b/core/services/gateway/connector/connector.go index 27db8fd44b6..7ee1572bb40 100644 --- a/core/services/gateway/connector/connector.go +++ b/core/services/gateway/connector/connector.go @@ -158,7 +158,7 @@ func (c *gatewayConnector) readLoop(gatewayState *gatewayState) { break } if err = msg.Validate(); err != nil { - c.lggr.Errorw("failed to validate message signature", "id", gatewayState.config.Id, "error", err) + c.lggr.Errorw("failed to validate message signature", "id", gatewayState.config.Id, "err", err) break } c.handler.HandleGatewayMessage(ctx, gatewayState.config.Id, msg) @@ -174,7 +174,7 @@ func (c *gatewayConnector) reconnectLoop(gatewayState *gatewayState) { for { conn, err := gatewayState.wsClient.Connect(ctx, gatewayState.url) if err != nil { - c.lggr.Errorw("connection error", "url", gatewayState.url, "error", err) + c.lggr.Errorw("connection error", "url", gatewayState.url, "err", err) } else { c.lggr.Infow("connected successfully", "url", gatewayState.url) closeCh := gatewayState.conn.Reset(conn) diff --git a/core/services/job/orm.go b/core/services/job/orm.go index 482d3d851e4..abaff5eb625 100644 --- a/core/services/job/orm.go +++ b/core/services/job/orm.go @@ -464,7 +464,7 @@ func (o *orm) CreateJob(jb *Job, qopts ...pg.QOpt) error { // ValidateKeyStoreMatch confirms that the key has a valid match in the keystore func ValidateKeyStoreMatch(spec *OCR2OracleSpec, keyStore keystore.Master, key string) error { - if spec.PluginType == types.Mercury { + if spec.PluginType == types.Mercury || spec.PluginType == types.Streams { _, err := keyStore.CSA().Get(key) if err != nil { return errors.Errorf("no CSA key matching: %q", key) diff --git a/core/services/job/spawner.go b/core/services/job/spawner.go index 5ed017b8743..9ee34f70f10 100644 --- a/core/services/job/spawner.go +++ b/core/services/job/spawner.go @@ -65,20 +65,20 @@ type ( Delegate interface { JobType() Type // BeforeJobCreated is only called once on first time job create. - BeforeJobCreated(spec Job) + BeforeJobCreated(Job) // ServicesForSpec returns services to be started and stopped for this // job. In case a given job type relies upon well-defined startup/shutdown // ordering for services, they are started in the order they are given // and stopped in reverse order. - ServicesForSpec(spec Job) ([]ServiceCtx, error) - AfterJobCreated(spec Job) - BeforeJobDeleted(spec Job) + ServicesForSpec(Job) ([]ServiceCtx, error) + AfterJobCreated(Job) + BeforeJobDeleted(Job) // OnDeleteJob will be called from within DELETE db transaction. Any db // commands issued within OnDeleteJob() should be performed first, before any // non-db side effects. This is required in order to guarantee mutual atomicity between // all tasks intended to happen during job deletion. For the same reason, the job will // not show up in the db within OnDeleteJob(), even though it is still actively running. - OnDeleteJob(spec Job, q pg.Queryer) error + OnDeleteJob(jb Job, q pg.Queryer) error } activeJob struct { diff --git a/core/services/keystore/keys/ocr2key/cosmos_keyring.go b/core/services/keystore/keys/ocr2key/cosmos_keyring.go index 490fa0cbfcb..19d475673c9 100644 --- a/core/services/keystore/keys/ocr2key/cosmos_keyring.go +++ b/core/services/keystore/keys/ocr2key/cosmos_keyring.go @@ -7,6 +7,7 @@ import ( "github.com/hdevalence/ed25519consensus" "github.com/pkg/errors" + "github.com/smartcontractkit/libocr/offchainreporting2/types" "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" "golang.org/x/crypto/blake2s" @@ -29,11 +30,11 @@ func newCosmosKeyring(material io.Reader) (*cosmosKeyring, error) { return &cosmosKeyring{pubKey: pubKey, privKey: privKey}, nil } -func (tk *cosmosKeyring) PublicKey() ocrtypes.OnchainPublicKey { - return []byte(tk.pubKey) +func (ckr *cosmosKeyring) PublicKey() ocrtypes.OnchainPublicKey { + return []byte(ckr.pubKey) } -func (tk *cosmosKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { +func (ckr *cosmosKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { rawReportContext := evmutil.RawReportContext(reportCtx) h, err := blake2s.New256(nil) if err != nil { @@ -49,48 +50,64 @@ func (tk *cosmosKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, repor return h.Sum(nil), nil } -func (tk *cosmosKeyring) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { - sigData, err := tk.reportToSigData(reportCtx, report) +func (ckr *cosmosKeyring) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { + sigData, err := ckr.reportToSigData(reportCtx, report) if err != nil { return nil, err } - signedMsg := ed25519.Sign(tk.privKey, sigData) + return ckr.signBlob(sigData) +} + +func (ckr *cosmosKeyring) Sign3(digest types.ConfigDigest, seqNr uint64, r ocrtypes.Report) (signature []byte, err error) { + panic("TODO") +} + +func (ckr *cosmosKeyring) signBlob(b []byte) ([]byte, error) { + signedMsg := ed25519.Sign(ckr.privKey, b) // match on-chain parsing (first 32 bytes are for pubkey, remaining are for signature) - return utils.ConcatBytes(tk.PublicKey(), signedMsg), nil + return utils.ConcatBytes(ckr.PublicKey(), signedMsg), nil +} + +func (ckr *cosmosKeyring) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { + hash, err := ckr.reportToSigData(reportCtx, report) + if err != nil { + return false + } + return ckr.verifyBlob(publicKey, hash, signature) +} + +func (ckr *cosmosKeyring) Verify3(publicKey ocrtypes.OnchainPublicKey, cd ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report, signature []byte) bool { + panic("TODO") } -func (tk *cosmosKeyring) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { +func (ckr *cosmosKeyring) verifyBlob(pubkey ocrtypes.OnchainPublicKey, b, sig []byte) bool { // Ed25519 signatures are always 64 bytes and the // public key (always prefixed, see Sign above) is always, // 32 bytes, so we always require the max signature length. - if len(signature) != tk.MaxSignatureLength() { + if len(sig) != ckr.MaxSignatureLength() { return false } - if len(publicKey) != ed25519.PublicKeySize { - return false - } - hash, err := tk.reportToSigData(reportCtx, report) - if err != nil { + if len(pubkey) != ed25519.PublicKeySize { return false } - return ed25519consensus.Verify(ed25519.PublicKey(publicKey), hash, signature[32:]) + return ed25519consensus.Verify(ed25519.PublicKey(pubkey), b, sig[32:]) } -func (tk *cosmosKeyring) MaxSignatureLength() int { +func (ckr *cosmosKeyring) MaxSignatureLength() int { // Reference: https://pkg.go.dev/crypto/ed25519 return ed25519.PublicKeySize + ed25519.SignatureSize // 32 + 64 } -func (tk *cosmosKeyring) Marshal() ([]byte, error) { - return tk.privKey.Seed(), nil +func (ckr *cosmosKeyring) Marshal() ([]byte, error) { + return ckr.privKey.Seed(), nil } -func (tk *cosmosKeyring) Unmarshal(in []byte) error { +func (ckr *cosmosKeyring) Unmarshal(in []byte) error { if len(in) != ed25519.SeedSize { return errors.Errorf("unexpected seed size, got %d want %d", len(in), ed25519.SeedSize) } privKey := ed25519.NewKeyFromSeed(in) - tk.privKey = privKey - tk.pubKey = privKey.Public().(ed25519.PublicKey) + ckr.privKey = privKey + ckr.pubKey = privKey.Public().(ed25519.PublicKey) return nil } diff --git a/core/services/keystore/keys/ocr2key/evm_keyring.go b/core/services/keystore/keys/ocr2key/evm_keyring.go index cc4076391b4..345c86a673f 100644 --- a/core/services/keystore/keys/ocr2key/evm_keyring.go +++ b/core/services/keystore/keys/ocr2key/evm_keyring.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/smartcontractkit/libocr/offchainreporting2/types" "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" ) @@ -26,12 +27,12 @@ func newEVMKeyring(material io.Reader) (*evmKeyring, error) { } // XXX: PublicKey returns the address of the public key not the public key itself -func (ok *evmKeyring) PublicKey() ocrtypes.OnchainPublicKey { - address := ok.signingAddress() +func (ekr *evmKeyring) PublicKey() ocrtypes.OnchainPublicKey { + address := ekr.signingAddress() return address[:] } -func (ok *evmKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) []byte { +func (ekr *evmKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) []byte { rawReportContext := evmutil.RawReportContext(reportCtx) sigData := crypto.Keccak256(report) sigData = append(sigData, rawReportContext[0][:]...) @@ -40,38 +41,54 @@ func (ok *evmKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report o return crypto.Keccak256(sigData) } -func (ok *evmKeyring) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { - return crypto.Sign(ok.reportToSigData(reportCtx, report), &ok.privateKey) +func (ekr *evmKeyring) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { + return ekr.signBlob(ekr.reportToSigData(reportCtx, report)) +} + +func (ekr *evmKeyring) Sign3(digest types.ConfigDigest, seqNr uint64, r ocrtypes.Report) (signature []byte, err error) { + panic("TODO") +} + +func (ekr *evmKeyring) signBlob(b []byte) (sig []byte, err error) { + return crypto.Sign(b, &ekr.privateKey) +} + +func (ekr *evmKeyring) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { + hash := ekr.reportToSigData(reportCtx, report) + return ekr.verifyBlob(publicKey, hash, signature) +} +func (ekr *evmKeyring) Verify3(publicKey ocrtypes.OnchainPublicKey, cd ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report, signature []byte) bool { + panic("TODO") } -func (ok *evmKeyring) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { - hash := ok.reportToSigData(reportCtx, report) - authorPubkey, err := crypto.SigToPub(hash, signature) +func (ekr *evmKeyring) verifyBlob(pubkey types.OnchainPublicKey, b, sig []byte) bool { + authorPubkey, err := crypto.SigToPub(b, sig) if err != nil { return false } authorAddress := crypto.PubkeyToAddress(*authorPubkey) - return bytes.Equal(publicKey[:], authorAddress[:]) + // no need for constant time compare since neither arg is sensitive + return bytes.Equal(pubkey[:], authorAddress[:]) } -func (ok *evmKeyring) MaxSignatureLength() int { +func (ekr *evmKeyring) MaxSignatureLength() int { return 65 } -func (ok *evmKeyring) signingAddress() common.Address { - return crypto.PubkeyToAddress(*(&ok.privateKey).Public().(*ecdsa.PublicKey)) +func (ekr *evmKeyring) signingAddress() common.Address { + return crypto.PubkeyToAddress(*(&ekr.privateKey).Public().(*ecdsa.PublicKey)) } -func (ok *evmKeyring) Marshal() ([]byte, error) { - return crypto.FromECDSA(&ok.privateKey), nil +func (ekr *evmKeyring) Marshal() ([]byte, error) { + return crypto.FromECDSA(&ekr.privateKey), nil } -func (ok *evmKeyring) Unmarshal(in []byte) error { +func (ekr *evmKeyring) Unmarshal(in []byte) error { privateKey, err := crypto.ToECDSA(in) if err != nil { return err } - ok.privateKey = *privateKey + ekr.privateKey = *privateKey return nil } diff --git a/core/services/keystore/keys/ocr2key/generic_key_bundle.go b/core/services/keystore/keys/ocr2key/generic_key_bundle.go index be401becfb3..2c5e4bd8559 100644 --- a/core/services/keystore/keys/ocr2key/generic_key_bundle.go +++ b/core/services/keystore/keys/ocr2key/generic_key_bundle.go @@ -18,6 +18,7 @@ import ( type ( keyring interface { ocrtypes.OnchainKeyring + OCR3SignerVerifier Marshal() ([]byte, error) Unmarshal(in []byte) error } @@ -92,10 +93,18 @@ func (kb *keyBundle[K]) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.R return kb.keyring.Sign(reportCtx, report) } +func (kb *keyBundle[K]) Sign3(digest ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report) (signature []byte, err error) { + return kb.keyring.Sign3(digest, seqNr, r) +} + func (kb *keyBundle[K]) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { return kb.keyring.Verify(publicKey, reportCtx, report, signature) } +func (kb *keyBundle[K]) Verify3(publicKey ocrtypes.OnchainPublicKey, cd ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report, signature []byte) bool { + return kb.keyring.Verify3(publicKey, cd, seqNr, r, signature) +} + // OnChainPublicKey returns public component of the keypair used on chain func (kb *keyBundle[K]) OnChainPublicKey() string { return hex.EncodeToString(kb.keyring.PublicKey()) diff --git a/core/services/keystore/keys/ocr2key/key_bundle.go b/core/services/keystore/keys/ocr2key/key_bundle.go index 79d8ad70d52..2c3a4bebeb0 100644 --- a/core/services/keystore/keys/ocr2key/key_bundle.go +++ b/core/services/keystore/keys/ocr2key/key_bundle.go @@ -14,12 +14,20 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/store/models" ) +type OCR3SignerVerifier interface { + Sign3(digest ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report) (signature []byte, err error) + Verify3(publicKey ocrtypes.OnchainPublicKey, cd ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report, signature []byte) bool +} + // nolint type KeyBundle interface { // OnchainKeyring is used for signing reports (groups of observations, verified onchain) ocrtypes.OnchainKeyring // OffchainKeyring is used for signing observations ocrtypes.OffchainKeyring + + OCR3SignerVerifier + ID() string ChainType() chaintype.ChainType Marshal() ([]byte, error) diff --git a/core/services/keystore/keys/ocr2key/solana_keyring.go b/core/services/keystore/keys/ocr2key/solana_keyring.go index aebe33e1d19..6ebb8d1c312 100644 --- a/core/services/keystore/keys/ocr2key/solana_keyring.go +++ b/core/services/keystore/keys/ocr2key/solana_keyring.go @@ -7,6 +7,7 @@ import ( "io" "github.com/ethereum/go-ethereum/crypto" + "github.com/smartcontractkit/libocr/offchainreporting2/types" "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" ) @@ -26,12 +27,12 @@ func newSolanaKeyring(material io.Reader) (*solanaKeyring, error) { } // XXX: PublicKey returns the evm-style address of the public key not the public key itself -func (ok *solanaKeyring) PublicKey() ocrtypes.OnchainPublicKey { - address := crypto.PubkeyToAddress(*(&ok.privateKey).Public().(*ecdsa.PublicKey)) +func (skr *solanaKeyring) PublicKey() ocrtypes.OnchainPublicKey { + address := crypto.PubkeyToAddress(*(&skr.privateKey).Public().(*ecdsa.PublicKey)) return address[:] } -func (ok *solanaKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) []byte { +func (skr *solanaKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) []byte { rawReportContext := evmutil.RawReportContext(reportCtx) h := sha256.New() h.Write([]byte{uint8(len(report))}) @@ -42,30 +43,47 @@ func (ok *solanaKeyring) reportToSigData(reportCtx ocrtypes.ReportContext, repor return h.Sum(nil) } -func (ok *solanaKeyring) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { - return crypto.Sign(ok.reportToSigData(reportCtx, report), &ok.privateKey) +func (skr *solanaKeyring) Sign(reportCtx ocrtypes.ReportContext, report ocrtypes.Report) ([]byte, error) { + return skr.signBlob(skr.reportToSigData(reportCtx, report)) } -func (ok *solanaKeyring) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { - hash := ok.reportToSigData(reportCtx, report) - authorPubkey, err := crypto.SigToPub(hash, signature) +func (skr *solanaKeyring) Sign3(digest types.ConfigDigest, seqNr uint64, r ocrtypes.Report) (signature []byte, err error) { + panic("TODO") +} + +func (skr *solanaKeyring) signBlob(b []byte) (sig []byte, err error) { + return crypto.Sign(b, &skr.privateKey) +} + +func (skr *solanaKeyring) Verify(publicKey ocrtypes.OnchainPublicKey, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signature []byte) bool { + hash := skr.reportToSigData(reportCtx, report) + return skr.verifyBlob(publicKey, hash, signature) +} + +func (skr *solanaKeyring) Verify3(publicKey ocrtypes.OnchainPublicKey, cd ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report, signature []byte) bool { + panic("TODO") +} + +func (skr *solanaKeyring) verifyBlob(pubkey types.OnchainPublicKey, b, sig []byte) bool { + authorPubkey, err := crypto.SigToPub(b, sig) if err != nil { return false } authorAddress := crypto.PubkeyToAddress(*authorPubkey) - return bytes.Equal(publicKey[:], authorAddress[:]) + // no need for constant time compare since neither arg is sensitive + return bytes.Equal(pubkey[:], authorAddress[:]) } -func (ok *solanaKeyring) MaxSignatureLength() int { +func (skr *solanaKeyring) MaxSignatureLength() int { return 65 } -func (ok *solanaKeyring) Marshal() ([]byte, error) { - return crypto.FromECDSA(&ok.privateKey), nil +func (skr *solanaKeyring) Marshal() ([]byte, error) { + return crypto.FromECDSA(&skr.privateKey), nil } -func (ok *solanaKeyring) Unmarshal(in []byte) error { +func (skr *solanaKeyring) Unmarshal(in []byte) error { privateKey, err := crypto.ToECDSA(in) - ok.privateKey = *privateKey + skr.privateKey = *privateKey return err } diff --git a/core/services/keystore/keys/starkkey/ocr2key.go b/core/services/keystore/keys/starkkey/ocr2key.go index 41ab3a4708d..bb7db4b523c 100644 --- a/core/services/keystore/keys/starkkey/ocr2key.go +++ b/core/services/keystore/keys/starkkey/ocr2key.go @@ -58,7 +58,6 @@ func (sk *OCR2Key) Sign(reportCtx types.ReportContext, report types.Report) ([]b if err != nil { return []byte{}, err } - r, s, err := caigo.Curve.Sign(hash, sk.priv) if err != nil { return []byte{}, err @@ -85,6 +84,10 @@ func (sk *OCR2Key) Sign(reportCtx types.ReportContext, report types.Report) ([]b return out, nil } +func (sk *OCR2Key) Sign3(digest types.ConfigDigest, seqNr uint64, r types.Report) (signature []byte, err error) { + panic("TODO") +} + func (sk *OCR2Key) Verify(publicKey types.OnchainPublicKey, reportCtx types.ReportContext, report types.Report, signature []byte) bool { // check valid signature length if len(signature) != sk.MaxSignatureLength() { @@ -120,6 +123,10 @@ func (sk *OCR2Key) Verify(publicKey types.OnchainPublicKey, reportCtx types.Repo return caigo.Curve.Verify(hash, r, s, keys[0].X, keys[0].Y) || caigo.Curve.Verify(hash, r, s, keys[1].X, keys[1].Y) } +func (sk *OCR2Key) Verify3(publicKey types.OnchainPublicKey, cd types.ConfigDigest, seqNr uint64, r types.Report, signature []byte) bool { + panic("TODO") +} + func (sk *OCR2Key) MaxSignatureLength() int { return 32 + 32 + 32 // publickey + r + s } diff --git a/core/services/ocr2/delegate.go b/core/services/ocr2/delegate.go index 16f02282afb..3755193fba4 100644 --- a/core/services/ocr2/delegate.go +++ b/core/services/ocr2/delegate.go @@ -68,6 +68,7 @@ import ( evmmercury "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" evmrelaytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/services/streams" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" "github.com/smartcontractkit/chainlink/v2/core/utils" @@ -431,6 +432,9 @@ func (d *Delegate) ServicesForSpec(jb job.Job) ([]job.ServiceCtx, error) { case types.Mercury: return d.newServicesMercury(ctx, lggr, jb, bootstrapPeers, kb, ocrDB, lc, ocrLogger) + case types.Streams: + return d.newServicesStreams(ctx, lggr, jb, bootstrapPeers, kb, ocrDB, lc, ocrLogger) + case types.Median: return d.newServicesMedian(ctx, lggr, jb, bootstrapPeers, kb, ocrDB, lc, ocrLogger) @@ -463,7 +467,7 @@ func (d *Delegate) ServicesForSpec(jb job.Job) ([]job.ServiceCtx, error) { func GetEVMEffectiveTransmitterID(jb *job.Job, chain legacyevm.Chain, lggr logger.SugaredLogger) (string, error) { spec := jb.OCR2OracleSpec - if spec.PluginType == types.Mercury { + if spec.PluginType == types.Mercury || spec.PluginType == types.Streams { return spec.TransmitterID.String, nil } @@ -729,6 +733,94 @@ func (d *Delegate) newServicesMercury( return mercuryServices, err2 } +func (d *Delegate) newServicesStreams( + ctx context.Context, + lggr logger.SugaredLogger, + jb job.Job, + bootstrapPeers []commontypes.BootstrapperLocator, + kb ocr2key.KeyBundle, + ocrDB *db, + lc ocrtypes.LocalConfig, + ocrLogger commontypes.Logger, +) ([]job.ServiceCtx, error) { + lggr = logger.Sugared(d.lggr.Named("Streams")) + spec := jb.OCR2OracleSpec + transmitterID := spec.TransmitterID.String + if len(transmitterID) != 64 { + return nil, errors.Errorf("ServicesForSpec: streams job type requires transmitter ID to be a 32-byte hex string, got: %q", transmitterID) + } + if _, err := hex.DecodeString(transmitterID); err != nil { + return nil, errors.Wrapf(err, "ServicesForSpec: streams job type requires transmitter ID to be a 32-byte hex string, got: %q", transmitterID) + } + + rid, err := spec.RelayID() + if err != nil { + return nil, ErrJobSpecNoRelayer{Err: err, PluginName: "streams"} + } + if rid.Network != relay.EVM { + return nil, fmt.Errorf("streams services: expected EVM relayer got %s", rid.Network) + } + relayer, err := d.RelayGetter.Get(rid) + if err != nil { + return nil, ErrRelayNotEnabled{Err: err, Relay: spec.Relay, PluginName: "streams"} + } + + provider, err2 := relayer.NewStreamsProvider(ctx, + types.RelayArgs{ + ExternalJobID: jb.ExternalJobID, + JobID: jb.ID, + ContractID: spec.ContractID, + New: d.isNewlyCreatedJob, + RelayConfig: spec.RelayConfig.Bytes(), + ProviderType: string(spec.PluginType), + }, types.PluginArgs{ + TransmitterID: transmitterID, + PluginConfig: spec.PluginConfig.Bytes(), + }) + if err2 != nil { + return nil, err2 + } + + streamsProvider, ok := provider.(types.StreamsProvider) + if !ok { + return nil, errors.New("could not coerce PluginProvider to streamsProvider") + } + + // chEnhancedTelem := make(chan ocrcommon.EnhancedTelemetryMercuryData, 100) + + // lloServices, err2 := llo.NewServices(jb, streamsProvider, d.pipelineRunner, runResults, lggr, oracleArgsNoPlugin, d.cfg.JobPipeline(), chEnhancedTelem, chain, d.mercuryORM, (mercuryutils.FeedID)(*spec.FeedID)) + + // if ocrcommon.ShouldCollectEnhancedTelemetryMercury(jb) { + // enhancedTelemService := ocrcommon.NewEnhancedTelemetryService(&jb, chEnhancedTelem, make(chan struct{}), d.monitoringEndpointGen.GenMonitoringEndpoint(rid.Network, rid.ChainID, spec.FeedID.String(), synchronization.EnhancedEAMercury), lggr.Named("EnhancedTelemetryMercury")) + // mercuryServices = append(mercuryServices, enhancedTelemService) + // } else { + // lggr.Infow("Enhanced telemetry is disabled for llo job", "job", jb.Name) + // } + + kr := streams.NewOnchainKeyring(kb) + + cfg := streams.DelegateConfig{ + Logger: lggr, + Queryer: pg.NewQ(d.db, d.lggr, d.cfg.Database()), + Runner: d.pipelineRunner, + + // TODO + BinaryNetworkEndpointFactory: d.peerWrapper.Peer2, + V2Bootstrappers: bootstrapPeers, + ContractTransmitter: streamsProvider.ContractTransmitter(), + ContractConfigTracker: streamsProvider.ContractConfigTracker(), + Database: ocrDB, + LocalConfig: lc, + MonitoringEndpoint: nil, // TODO + OffchainConfigDigester: streamsProvider.OffchainConfigDigester(), + OffchainKeyring: kb, + OnchainKeyring: kr, + OCRLogger: ocrLogger, + } + srv := streams.NewDelegate(cfg) + return []job.ServiceCtx{srv}, nil +} + func (d *Delegate) newServicesMedian( ctx context.Context, lggr logger.SugaredLogger, diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go index c7f6884426f..aa519ce3e82 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go @@ -540,7 +540,7 @@ func (r *logRecoverer) selectFilterBatch(filters []upkeepFilter) []upkeepFilter for len(results) < batchSize && len(filters) != 0 { i, err := r.randIntn(len(filters)) if err != nil { - r.lggr.Debugw("error generating random number", "error", err.Error()) + r.lggr.Debugw("error generating random number", "err", err.Error()) continue } results = append(results, filters[i]) diff --git a/core/services/ocr2/plugins/s4/plugin_test.go b/core/services/ocr2/plugins/s4/plugin_test.go index e2b5d21b847..56cfbc32489 100644 --- a/core/services/ocr2/plugins/s4/plugin_test.go +++ b/core/services/ocr2/plugins/s4/plugin_test.go @@ -222,7 +222,7 @@ func TestPlugin_ShouldAcceptFinalizedReport(t *testing.T) { }) - t.Run("error", func(t *testing.T) { + t.Run("err", func(t *testing.T) { testErr := errors.New("some error") rows := generateTestRows(t, 1, time.Minute) orm.On("Update", mock.Anything, mock.Anything).Return(testErr).Once() diff --git a/core/services/ocr2/plugins/streams/config/config.go b/core/services/ocr2/plugins/streams/config/config.go new file mode 100644 index 00000000000..cb0b15d7e57 --- /dev/null +++ b/core/services/ocr2/plugins/streams/config/config.go @@ -0,0 +1,52 @@ +// config is a separate package so that we can validate +// the config in other packages, for example in job at job create time. + +package config + +import ( + "errors" + "fmt" + "net/url" + "regexp" + + pkgerrors "github.com/pkg/errors" + + "github.com/smartcontractkit/chainlink/v2/core/utils" +) + +type PluginConfig struct { + RawServerURL string `json:"serverURL" toml:"serverURL"` + ServerPubKey utils.PlainHexBytes `json:"serverPubKey" toml:"serverPubKey"` +} + +func (p PluginConfig) Validate() (merr error) { + if p.RawServerURL == "" { + merr = errors.New("streams: ServerURL must be specified") + } else { + var normalizedURI string + if schemeRegexp.MatchString(p.RawServerURL) { + normalizedURI = p.RawServerURL + } else { + normalizedURI = fmt.Sprintf("wss://%s", p.RawServerURL) + } + uri, err := url.ParseRequestURI(normalizedURI) + if err != nil { + merr = pkgerrors.Wrap(err, "streams: invalid value for ServerURL") + } else if uri.Scheme != "wss" { + merr = pkgerrors.Errorf(`streams: invalid scheme specified for MercuryServer, got: %q (scheme: %q) but expected a websocket url e.g. "192.0.2.2:4242" or "wss://192.0.2.2:4242"`, p.RawServerURL, uri.Scheme) + } + } + + if len(p.ServerPubKey) != 32 { + merr = errors.Join(merr, errors.New("streams: ServerPubKey is required and must be a 32-byte hex string")) + } + + return merr +} + +var schemeRegexp = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`) +var wssRegexp = regexp.MustCompile(`^wss://`) + +func (p PluginConfig) ServerURL() string { + return wssRegexp.ReplaceAllString(p.RawServerURL, "") +} diff --git a/core/services/ocr2/plugins/streams/helpers_test.go b/core/services/ocr2/plugins/streams/helpers_test.go new file mode 100644 index 00000000000..5cf9f6f8eb0 --- /dev/null +++ b/core/services/ocr2/plugins/streams/helpers_test.go @@ -0,0 +1,463 @@ +package streams_test + +import ( + "context" + "crypto/ed25519" + "encoding/binary" + "errors" + "fmt" + "math/big" + "net" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/smartcontractkit/wsrpc" + "github.com/smartcontractkit/wsrpc/credentials" + "github.com/smartcontractkit/wsrpc/peer" + + "github.com/smartcontractkit/libocr/offchainreporting2/chains/evmutil" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest/heavyweight" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/keystest" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/csakey" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ocr2key" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/validate" + "github.com/smartcontractkit/chainlink/v2/core/services/ocrbootstrap" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" + "github.com/smartcontractkit/chainlink/v2/core/store/models" + "github.com/smartcontractkit/chainlink/v2/core/utils" +) + +var _ pb.MercuryServer = &mercuryServer{} + +type request struct { + pk credentials.StaticSizedPublicKey + req *pb.TransmitRequest +} + +type mercuryServer struct { + privKey ed25519.PrivateKey + reqsCh chan request + t *testing.T + buildReport func() []byte +} + +func NewMercuryServer(t *testing.T, privKey ed25519.PrivateKey, reqsCh chan request, buildReport func() []byte) *mercuryServer { + return &mercuryServer{privKey, reqsCh, t, buildReport} +} + +func (s *mercuryServer) Transmit(ctx context.Context, req *pb.TransmitRequest) (*pb.TransmitResponse, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, errors.New("could not extract public key") + } + r := request{p.PublicKey, req} + s.reqsCh <- r + + return &pb.TransmitResponse{ + Code: 1, + Error: "", + }, nil +} + +func (s *mercuryServer) LatestReport(ctx context.Context, lrr *pb.LatestReportRequest) (*pb.LatestReportResponse, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, errors.New("could not extract public key") + } + s.t.Logf("mercury server got latest report from %x for feed id 0x%x", p.PublicKey, lrr.FeedId) + + out := new(pb.LatestReportResponse) + out.Report = new(pb.Report) + out.Report.FeedId = lrr.FeedId + + report := s.buildReport() + payload, err := mercury.PayloadTypes.Pack(evmutil.RawReportContext(ocrtypes.ReportContext{}), report, [][32]byte{}, [][32]byte{}, [32]byte{}) + if err != nil { + panic(err) + } + out.Report.Payload = payload + return out, nil +} + +func startMercuryServer(t *testing.T, srv *mercuryServer, pubKeys []ed25519.PublicKey) (serverURL string) { + // Set up the wsrpc server + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("[MAIN] failed to listen: %v", err) + } + serverURL = lis.Addr().String() + s := wsrpc.NewServer(wsrpc.Creds(srv.privKey, pubKeys)) + + // Register mercury implementation with the wsrpc server + pb.RegisterMercuryServer(s, srv) + + // Start serving + go s.Serve(lis) + t.Cleanup(s.Stop) + + return +} + +type Job struct { + name string + id [32]byte + baseBenchmarkPrice *big.Int + baseBid *big.Int + baseAsk *big.Int +} + +func randomFeedID(version uint16) [32]byte { + id := [32]byte(utils.NewHash()) + binary.BigEndian.PutUint16(id[:2], version) + return id +} + +type Node struct { + App chainlink.Application + ClientPubKey credentials.StaticSizedPublicKey + KeyBundle ocr2key.KeyBundle +} + +func (node *Node) AddJob(t *testing.T, spec string) { + c := node.App.GetConfig() + job, err := validate.ValidatedOracleSpecToml(c.OCR2(), c.Insecure(), spec) + require.NoError(t, err) + err = node.App.AddJobV2(testutils.Context(t), &job) + require.NoError(t, err) +} + +func (node *Node) AddBootstrapJob(t *testing.T, spec string) { + job, err := ocrbootstrap.ValidatedBootstrapSpecToml(spec) + require.NoError(t, err) + err = node.App.AddJobV2(testutils.Context(t), &job) + require.NoError(t, err) +} + +func setupNode( + t *testing.T, + port int, + dbName string, + backend *backends.SimulatedBackend, + csaKey csakey.KeyV2, +) (app chainlink.Application, peerID string, clientPubKey credentials.StaticSizedPublicKey, ocr2kb ocr2key.KeyBundle, observedLogs *observer.ObservedLogs) { + k := big.NewInt(int64(port)) // keys unique to port + p2pKey := p2pkey.MustNewV2XXXTestingOnly(k) + rdr := keystest.NewRandReaderFromSeed(int64(port)) + ocr2kb = ocr2key.MustNewInsecure(rdr, chaintype.EVM) + + p2paddresses := []string{fmt.Sprintf("127.0.0.1:%d", port)} + + config, _ := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { + // [JobPipeline] + // MaxSuccessfulRuns = 0 + c.JobPipeline.MaxSuccessfulRuns = ptr(uint64(0)) + + // [Feature] + // UICSAKeys=true + // LogPoller = true + // FeedsManager = false + c.Feature.UICSAKeys = ptr(true) + c.Feature.LogPoller = ptr(true) + c.Feature.FeedsManager = ptr(false) + + // [OCR] + // Enabled = false + c.OCR.Enabled = ptr(false) + + // [OCR2] + // Enabled = true + c.OCR2.Enabled = ptr(true) + + // [P2P] + // PeerID = '$PEERID' + // TraceLogging = true + c.P2P.PeerID = ptr(p2pKey.PeerID()) + c.P2P.TraceLogging = ptr(true) + + // [P2P.V2] + // Enabled = true + // AnnounceAddresses = ['$EXT_IP:17775'] + // ListenAddresses = ['127.0.0.1:17775'] + // DeltaDial = 500ms + // DeltaReconcile = 5s + c.P2P.V2.Enabled = ptr(true) + c.P2P.V2.AnnounceAddresses = &p2paddresses + c.P2P.V2.ListenAddresses = &p2paddresses + c.P2P.V2.DeltaDial = models.MustNewDuration(500 * time.Millisecond) + c.P2P.V2.DeltaReconcile = models.MustNewDuration(5 * time.Second) + }) + + lggr, observedLogs := logger.TestLoggerObserved(t, zapcore.DebugLevel) + app = cltest.NewApplicationWithConfigV2OnSimulatedBlockchain(t, config, backend, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) + err := app.Start(testutils.Context(t)) + require.NoError(t, err) + + t.Cleanup(func() { + assert.NoError(t, app.Stop()) + }) + + return app, p2pKey.PeerID().Raw(), csaKey.StaticSizedPublicKey(), ocr2kb, observedLogs +} + +func ptr[T any](t T) *T { return &t } + +func addBootstrapJob(t *testing.T, bootstrapNode Node, chainID *big.Int, verifierAddress common.Address, name string) { + bootstrapNode.AddBootstrapJob(t, fmt.Sprintf(` +type = "bootstrap" +relay = "evm" +schemaVersion = 1 +name = "boot-%s" +contractID = "%s" +contractConfigTrackerPollInterval = "1s" + +[relayConfig] +chainID = %d + `, name, verifierAddress, chainID)) +} + +func addV1MercuryJob( + t *testing.T, + node Node, + i int, + verifierAddress common.Address, + bootstrapPeerID string, + bootstrapNodePort int, + bmBridge, + bidBridge, + askBridge, + serverURL string, + serverPubKey, + clientPubKey ed25519.PublicKey, + feedName string, + feedID [32]byte, + chainID *big.Int, + fromBlock int, +) { + node.AddJob(t, fmt.Sprintf(` +type = "offchainreporting2" +schemaVersion = 1 +name = "mercury-%[1]d-%[14]s" +forwardingAllowed = false +maxTaskDuration = "1s" +contractID = "%[2]s" +feedID = "0x%[11]x" +contractConfigTrackerPollInterval = "1s" +ocrKeyBundleID = "%[3]s" +p2pv2Bootstrappers = [ + "%[4]s" +] +relay = "evm" +pluginType = "mercury" +transmitterID = "%[10]x" +observationSource = """ + // Benchmark Price + price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + price1_parse [type=jsonparse path="result"]; + price1_multiply [type=multiply times=100000000 index=0]; + + price1 -> price1_parse -> price1_multiply; + + // Bid + bid [type=bridge name="%[6]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + bid_parse [type=jsonparse path="result"]; + bid_multiply [type=multiply times=100000000 index=1]; + + bid -> bid_parse -> bid_multiply; + + // Ask + ask [type=bridge name="%[7]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + ask_parse [type=jsonparse path="result"]; + ask_multiply [type=multiply times=100000000 index=2]; + + ask -> ask_parse -> ask_multiply; +""" + +[pluginConfig] +serverURL = "%[8]s" +serverPubKey = "%[9]x" +initialBlockNumber = %[13]d + +[relayConfig] +chainID = %[12]d + + `, + i, + verifierAddress, + node.KeyBundle.ID(), + fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), + bmBridge, + bidBridge, + askBridge, + serverURL, + serverPubKey, + clientPubKey, + feedID, + chainID, + fromBlock, + feedName, + )) +} + +func addV2MercuryJob( + t *testing.T, + node Node, + i int, + verifierAddress common.Address, + bootstrapPeerID string, + bootstrapNodePort int, + bmBridge, + serverURL string, + serverPubKey, + clientPubKey ed25519.PublicKey, + feedName string, + feedID [32]byte, + linkFeedID [32]byte, + nativeFeedID [32]byte, +) { + node.AddJob(t, fmt.Sprintf(` +type = "offchainreporting2" +schemaVersion = 1 +name = "mercury-%[1]d-%[10]s" +forwardingAllowed = false +maxTaskDuration = "1s" +contractID = "%[2]s" +feedID = "0x%[9]x" +contractConfigTrackerPollInterval = "1s" +ocrKeyBundleID = "%[3]s" +p2pv2Bootstrappers = [ + "%[4]s" +] +relay = "evm" +pluginType = "mercury" +transmitterID = "%[8]x" +observationSource = """ + // Benchmark Price + price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + price1_parse [type=jsonparse path="result"]; + price1_multiply [type=multiply times=100000000 index=0]; + + price1 -> price1_parse -> price1_multiply; +""" + +[pluginConfig] +serverURL = "%[6]s" +serverPubKey = "%[7]x" +linkFeedID = "0x%[11]x" +nativeFeedID = "0x%[12]x" + +[relayConfig] +chainID = 1337 + `, + i, + verifierAddress, + node.KeyBundle.ID(), + fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), + bmBridge, + serverURL, + serverPubKey, + clientPubKey, + feedID, + feedName, + linkFeedID, + nativeFeedID, + )) +} + +func addV3MercuryJob( + t *testing.T, + node Node, + i int, + verifierAddress common.Address, + bootstrapPeerID string, + bootstrapNodePort int, + bmBridge, + bidBridge, + askBridge, + serverURL string, + serverPubKey, + clientPubKey ed25519.PublicKey, + feedName string, + feedID [32]byte, + linkFeedID [32]byte, + nativeFeedID [32]byte, +) { + node.AddJob(t, fmt.Sprintf(` +type = "offchainreporting2" +schemaVersion = 1 +name = "mercury-%[1]d-%[12]s" +forwardingAllowed = false +maxTaskDuration = "1s" +contractID = "%[2]s" +feedID = "0x%[11]x" +contractConfigTrackerPollInterval = "1s" +ocrKeyBundleID = "%[3]s" +p2pv2Bootstrappers = [ + "%[4]s" +] +relay = "evm" +pluginType = "mercury" +transmitterID = "%[10]x" +observationSource = """ + // Benchmark Price + price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + price1_parse [type=jsonparse path="result"]; + price1_multiply [type=multiply times=100000000 index=0]; + + price1 -> price1_parse -> price1_multiply; + + // Bid + bid [type=bridge name="%[6]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + bid_parse [type=jsonparse path="result"]; + bid_multiply [type=multiply times=100000000 index=1]; + + bid -> bid_parse -> bid_multiply; + + // Ask + ask [type=bridge name="%[7]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; + ask_parse [type=jsonparse path="result"]; + ask_multiply [type=multiply times=100000000 index=2]; + + ask -> ask_parse -> ask_multiply; +""" + +[pluginConfig] +serverURL = "%[8]s" +serverPubKey = "%[9]x" +linkFeedID = "0x%[13]x" +nativeFeedID = "0x%[14]x" + +[relayConfig] +chainID = 1337 + `, + i, + verifierAddress, + node.KeyBundle.ID(), + fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), + bmBridge, + bidBridge, + askBridge, + serverURL, + serverPubKey, + clientPubKey, + feedID, + feedName, + linkFeedID, + nativeFeedID, + )) +} diff --git a/core/services/ocr2/plugins/streams/integration_test.go b/core/services/ocr2/plugins/streams/integration_test.go new file mode 100644 index 00000000000..fb280164708 --- /dev/null +++ b/core/services/ocr2/plugins/streams/integration_test.go @@ -0,0 +1,363 @@ +package streams_test + +import ( + "crypto/ed25519" + "encoding/hex" + "fmt" + "math/big" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/hashicorp/consul/sdk/freeport" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + "github.com/smartcontractkit/libocr/gethwrappers2/ocrconfigurationstoreevmsimple" + "github.com/smartcontractkit/wsrpc/credentials" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/csakey" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" + "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" +) + +var ( + f = uint8(1) + n = 4 // number of nodes + multiplier int64 = 100000000 +) + +func setupBlockchain(t *testing.T) (*bind.TransactOpts, *backends.SimulatedBackend, *ocrconfigurationstoreevmsimple.OCRConfigurationStoreEVMSimple, common.Address) { + steve := testutils.MustNewSimTransactor(t) // config contract deployer and owner + genesisData := core.GenesisAlloc{steve.From: {Balance: assets.Ether(1000).ToInt()}} + backend := cltest.NewSimulatedBackend(t, genesisData, uint32(ethconfig.Defaults.Miner.GasCeil)) + backend.Commit() // ensure starting block number at least 1 + stopMining := cltest.Mine(backend, 1*time.Second) // Should be greater than deltaRound since we cannot access old blocks on simulated blockchain + t.Cleanup(stopMining) + + // Deploy contracts + configAddress, _, configContract, err := ocrconfigurationstoreevmsimple.DeployOCRConfigurationStoreEVMSimple(steve, backend) + require.NoError(t, err) + + // linkTokenAddress, _, linkToken, err := link_token_interface.DeployLinkToken(steve, backend) + // require.NoError(t, err) + // _, err = linkToken.Transfer(steve, steve.From, big.NewInt(1000)) + // require.NoError(t, err) + // nativeTokenAddress, _, nativeToken, err := link_token_interface.DeployLinkToken(steve, backend) + // require.NoError(t, err) + // _, err = nativeToken.Transfer(steve, steve.From, big.NewInt(1000)) + // require.NoError(t, err) + // verifierProxyAddr, _, verifierProxy, err := verifier_proxy.DeployVerifierProxy(steve, backend, common.Address{}) // zero address for access controller disables access control + // require.NoError(t, err) + // verifierAddress, _, verifier, err := verifier.DeployVerifier(steve, backend, verifierProxyAddr) + // require.NoError(t, err) + // _, err = verifierProxy.InitializeVerifier(steve, verifierAddress) + // require.NoError(t, err) + // rewardManagerAddr, _, rewardManager, err := reward_manager.DeployRewardManager(steve, backend, linkTokenAddress) + // require.NoError(t, err) + // feeManagerAddr, _, _, err := fee_manager.DeployFeeManager(steve, backend, linkTokenAddress, nativeTokenAddress, verifierProxyAddr, rewardManagerAddr) + // require.NoError(t, err) + // _, err = verifierProxy.SetFeeManager(steve, feeManagerAddr) + // require.NoError(t, err) + // _, err = rewardManager.SetFeeManager(steve, feeManagerAddr) + // require.NoError(t, err) + + backend.Commit() + + return steve, backend, configContract, configAddress +} + +func detectPanicLogs(t *testing.T, logObservers []*observer.ObservedLogs) { + var panicLines []string + for _, observedLogs := range logObservers { + panicLogs := observedLogs.Filter(func(e observer.LoggedEntry) bool { + return e.Level >= zapcore.DPanicLevel + }) + for _, log := range panicLogs.All() { + line := fmt.Sprintf("%v\t%s\t%s\t%s\t%s", log.Time.Format(time.RFC3339), log.Level.CapitalString(), log.LoggerName, log.Caller.TrimmedPath(), log.Message) + panicLines = append(panicLines, line) + } + } + if len(panicLines) > 0 { + t.Errorf("Found logs with DPANIC or higher level:\n%s", strings.Join(panicLines, "\n")) + } +} + +// type mercuryServer struct { +// privKey ed25519.PrivateKey +// reqsCh chan request +// t *testing.T +// } + +// func NewMercuryServer(t *testing.T, privKey ed25519.PrivateKey, reqsCh chan request) *mercuryServer { +// return &mercuryServer{privKey, reqsCh, t} +// } + +// func (s *mercuryServer) Transmit(ctx context.Context, req *pb.TransmitRequest) (*pb.TransmitResponse, error) { +// p, ok := peer.FromContext(ctx) +// if !ok { +// return nil, errors.New("could not extract public key") +// } +// r := request{p.PublicKey, req} +// s.reqsCh <- r + +// return &pb.TransmitResponse{ +// Code: 1, +// Error: "", +// }, nil +// } + +// func (s *mercuryServer) LatestReport(ctx context.Context, lrr *pb.LatestReportRequest) (*pb.LatestReportResponse, error) { +// panic("not needed for llo") +// } + +func TestIntegration_Streams(t *testing.T) { + // TODO: + + t.Parallel() + + var logObservers []*observer.ObservedLogs + t.Cleanup(func() { + detectPanicLogs(t, logObservers) + }) + const fromBlock = 1 // cannot use zero, start from block 1 + // testStartTimeStamp := uint32(time.Now().Unix()) + + reqs := make(chan request) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(-1)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, ed25519.PrivateKey(serverKey.Raw()), reqs, nil) + clientCSAKeys := make([]csakey.KeyV2, n+1) + clientPubKeys := make([]ed25519.PublicKey, n+1) + for i := 0; i < n+1; i++ { + k := big.NewInt(int64(i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + serverURL := startMercuryServer(t, srv, clientPubKeys) + chainID := testutils.SimulatedChainID + + steve, backend, configContract, configAddress := setupBlockchain(t) + // TODO + + // Setup bootstrap + oracle nodes + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, observedLogs := setupNode(t, bootstrapNodePort, "bootstrap_mercury", backend, clientCSAKeys[n]) + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + logObservers = append(logObservers, observedLogs) + + // Set up n oracles + var ( + oracles []confighelper.OracleIdentityExtra + nodes []Node + ) + ports := freeport.GetN(t, n) + for i := 0; i < n; i++ { + app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_streams_%d", i), backend, clientCSAKeys[i]) + + nodes = append(nodes, Node{ + app, transmitter, kb, + }) + offchainPublicKey, _ := hex.DecodeString(strings.TrimPrefix(kb.OnChainPublicKey(), "0x")) + oracles = append(oracles, confighelper.OracleIdentityExtra{ + OracleIdentity: confighelper.OracleIdentity{ + OnchainPublicKey: offchainPublicKey, + TransmitAccount: ocr2types.Account(fmt.Sprintf("%x", transmitter[:])), + OffchainPublicKey: kb.OffchainPublicKey(), + PeerID: peerID, + }, + ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), + }) + logObservers = append(logObservers, observedLogs) + } + + addBootstrapJob(t, bootstrapNode, chainID, configAddress, "job-1") + + // createBridge := func(name string, i int, p *big.Int, borm bridges.ORM) (bridgeName string) { + // bridge := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + // b, err := io.ReadAll(req.Body) + // require.NoError(t, err) + // require.Equal(t, `{"data":{"from":"ETH","to":"USD"}}`, string(b)) + + // res.WriteHeader(http.StatusOK) + // val := decimal.NewFromBigInt(p, 0).Div(decimal.NewFromInt(multiplier)).Add(decimal.NewFromInt(int64(i)).Div(decimal.NewFromInt(100))).String() + // resp := fmt.Sprintf(`{"result": %s}`, val) + // _, err = res.Write([]byte(resp)) + // require.NoError(t, err) + // })) + // t.Cleanup(bridge.Close) + // u, _ := url.Parse(bridge.URL) + // bridgeName = fmt.Sprintf("bridge-%s-%d", name, i) + // require.NoError(t, borm.CreateBridgeType(&bridges.BridgeType{ + // Name: bridges.BridgeName(bridgeName), + // URL: models.WebURL(*u), + // })) + + // return bridgeName + // } + + // Add OCR jobs - one per feed on each node + for i, node := range nodes { + addStreamsJob( + t, + node, + configAddress, + bootstrapPeerID, + bootstrapNodePort, + serverURL, + serverPubKey, + clientPubKeys[i], + "feed-1", + chainID, + fromBlock, + ) + } + + // Setup config on contract + rawOnchainConfig := streams.OnchainConfig{} + // TODO: Move away from JSON + onchainConfig, err := (&streams.JSONOnchainConfigCodec{}).Encode(rawOnchainConfig) + require.NoError(t, err) + + rawReportingPluginConfig := streams.OffchainConfig{} + reportingPluginConfig, err := rawReportingPluginConfig.Encode() + require.NoError(t, err) + + signers, _, _, onchainConfig, _, _, err := ocr3confighelper.ContractSetConfigArgsForTestsMercuryV02( + 2*time.Second, // DeltaProgress + 20*time.Second, // DeltaResend + 400*time.Millisecond, // DeltaInitial + 100*time.Millisecond, // DeltaRound + 0, // DeltaGrace + 300*time.Millisecond, // DeltaCertifiedCommitRequest + 1*time.Minute, // DeltaStage + 100, // rMax + []int{len(nodes)}, // S + oracles, + reportingPluginConfig, // reportingPluginConfig []byte, + 250*time.Millisecond, // Max duration observation + int(f), // f + onchainConfig, + ) + + require.NoError(t, err) + signerAddresses, err := evm.OnchainPublicKeyToAddress(signers) + require.NoError(t, err) + + offchainTransmitters := make([][32]byte, n) + for i := 0; i < n; i++ { + offchainTransmitters[i] = nodes[i].ClientPubKey + } + + cfg := ocrconfigurationstoreevmsimple.OCRConfigurationStoreEVMSimpleConfigurationEVMSimple{ + signerAddresses, + offchainTransmitters, + f, + onchainConfig, + offchainConfigVersion, + offchainConfig, + nil, + } + _, err = configContract.AddConfig( + steve, + cfg, + ) + require.NoError(t, err) + backend.Commit() + + // Bury it with finality depth + ch, err := bootstrapNode.App.GetRelayers().LegacyEVMChains().Get(testutils.SimulatedChainID.String()) + require.NoError(t, err) + finalityDepth := ch.Config().EVM().FinalityDepth() + for i := 0; i < int(finalityDepth); i++ { + backend.Commit() + } + + t.Run("receives at least one report per feed from each oracle when EAs are at 100% reliability", func(t *testing.T) { + // Expect at least one report per feed from each oracle + seen := make(map[credentials.StaticSizedPublicKey]struct{}) + + for req := range reqs { + v := make(map[string]interface{}) + err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) + require.NoError(t, err) + report, exists := v["report"] + if !exists { + t.Fatalf("expected payload %#v to contain 'report'", v) + } + + assert.Equal(t, "foo", report) + + seen[req.pk] = struct{}{} + if len(seen) == n { + t.Logf("all oracles reported") + break // saw all oracles; success! + } + } + }) +} + +func addStreamsJob( + t *testing.T, + node Node, + verifierAddress common.Address, + bootstrapPeerID string, + bootstrapNodePort int, + serverURL string, + serverPubKey, + clientPubKey ed25519.PublicKey, + jobName string, + chainID *big.Int, + fromBlock int, +) { + node.AddJob(t, fmt.Sprintf(` +type = "offchainreporting2" +schemaVersion = 1 +name = "%[1]s" +forwardingAllowed = false +maxTaskDuration = "1s" +contractID = "%[2]s" +contractConfigTrackerPollInterval = "1s" +ocrKeyBundleID = "%[3]s" +p2pv2Bootstrappers = [ + "%[4]s" +] +relay = "evm" +pluginType = "streams" +transmitterID = "%[5]x" + +[pluginConfig] +serverURL = "%[6]s" +serverPubKey = "%[7]x" +fromBlock = %[8]d + +[relayConfig] +chainID = %[9]d + + `, + jobName, + verifierAddress, + node.KeyBundle.ID(), + fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), + clientPubKey, + serverURL, + serverPubKey, + fromBlock, + chainID, + )) +} diff --git a/core/services/ocr2/plugins/streams/plugin.go b/core/services/ocr2/plugins/streams/plugin.go new file mode 100644 index 00000000000..d8133764ae1 --- /dev/null +++ b/core/services/ocr2/plugins/streams/plugin.go @@ -0,0 +1 @@ +package streams diff --git a/core/services/ocr2/validate/validate.go b/core/services/ocr2/validate/validate.go index bb9bb03a8ac..c536e035869 100644 --- a/core/services/ocr2/validate/validate.go +++ b/core/services/ocr2/validate/validate.go @@ -17,6 +17,7 @@ import ( dkgconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/dkg/config" mercuryconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" ocr2vrfconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2vrf/config" + streamsconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/streams/config" "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" "github.com/smartcontractkit/chainlink/v2/core/services/relay" ) @@ -114,6 +115,8 @@ func validateSpec(tree *toml.Tree, spec job.Job) error { return nil case types.Mercury: return validateOCR2MercurySpec(spec.OCR2OracleSpec.PluginConfig, *spec.OCR2OracleSpec.FeedID) + case types.Streams: + return validateOCR2StreamsSpec(spec.OCR2OracleSpec.PluginConfig) case types.GenericPlugin: return validateOCR2GenericPluginSpec(spec.OCR2OracleSpec.PluginConfig) case "": @@ -255,3 +258,12 @@ func validateOCR2MercurySpec(jsonConfig job.JSONConfig, feedId [32]byte) error { } return pkgerrors.Wrap(mercuryconfig.ValidatePluginConfig(pluginConfig, feedId), "Mercury PluginConfig is invalid") } + +func validateOCR2StreamsSpec(jsonConfig job.JSONConfig) error { + var pluginConfig streamsconfig.PluginConfig + err := json.Unmarshal(jsonConfig.Bytes(), &pluginConfig) + if err != nil { + return pkgerrors.Wrap(err, "error while unmarshaling plugin config") + } + return pkgerrors.Wrap(pluginConfig.Validate(), "Streams PluginConfig is invalid") +} diff --git a/core/services/relay/evm/evm.go b/core/services/relay/evm/evm.go index aea704adacf..61a706dbdfa 100644 --- a/core/services/relay/evm/evm.go +++ b/core/services/relay/evm/evm.go @@ -22,6 +22,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/services" commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + datastreams "github.com/smartcontractkit/chainlink-data-streams/streams" txmgrcommon "github.com/smartcontractkit/chainlink/v2/common/txmgr" txm "github.com/smartcontractkit/chainlink/v2/core/chains/evm/txmgr" @@ -31,6 +32,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/keystore" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ethkey" mercuryconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" + streamsconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/streams/config" "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" "github.com/smartcontractkit/chainlink/v2/core/services/pg" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" @@ -42,6 +44,7 @@ import ( reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/services/streams" ) var _ commontypes.Relayer = &Relayer{} //nolint:staticcheck @@ -188,11 +191,56 @@ func (r *Relayer) NewMercuryProvider(rargs commontypes.RelayArgs, pargs commonty default: return nil, fmt.Errorf("invalid feed version %d", feedID.Version()) } - transmitter := mercury.NewTransmitter(lggr, cw.ContractConfigTracker(), client, privKey.PublicKey, rargs.JobID, *relayConfig.FeedID, r.db, r.pgCfg, transmitterCodec) + transmitter := mercury.NewTransmitter(lggr, client, privKey.PublicKey, rargs.JobID, *relayConfig.FeedID, r.db, r.pgCfg, transmitterCodec) return NewMercuryProvider(cw, r.chainReader, NewMercuryChainReader(r.chain.HeadTracker()), transmitter, reportCodecV1, reportCodecV2, reportCodecV3, lggr), nil } +func (r *Relayer) NewStreamsProvider(rargs commontypes.RelayArgs, pargs commontypes.PluginArgs) (commontypes.StreamsProvider, error) { + // TODO + relayOpts := types.NewRelayOpts(rargs) + relayConfig, err := relayOpts.RelayConfig() + if err != nil { + return nil, fmt.Errorf("failed to get relay config: %w", err) + } + + var streamsConfig streamsconfig.PluginConfig + if err := json.Unmarshal(pargs.PluginConfig, &streamsConfig); err != nil { + return nil, pkgerrors.WithStack(err) + } + if err := streamsConfig.Validate(); err != nil { + return nil, err + } + + if relayConfig.ChainID.String() != r.chain.ID().String() { + return nil, fmt.Errorf("internal error: chain id in spec does not match this relayer's chain: have %s expected %s", relayConfig.ChainID.String(), r.chain.ID().String()) + } + configWatcher, err := newConfigProvider(r.lggr, r.chain, relayOpts, r.eventBroadcaster) + if err != nil { + return nil, pkgerrors.WithStack(err) + } + + if !relayConfig.EffectiveTransmitterID.Valid { + return nil, pkgerrors.New("EffectiveTransmitterID must be specified") + } + privKey, err := r.ks.CSA().Get(relayConfig.EffectiveTransmitterID.String) + if err != nil { + return nil, pkgerrors.Wrap(err, "failed to get CSA key for mercury connection") + } + + client, err := r.mercuryPool.Checkout(context.Background(), privKey, streamsConfig.ServerPubKey, streamsConfig.ServerURL()) + if err != nil { + return nil, err + } + + // FIXME + // transmitter := streamsNewTransmitter(r.lggr, configWatcher.ContractConfigTracker(), client, privKey.PublicKey, rargs.JobID, r.db, r.pgCfg) + transmitter := streams.NewTransmitter(r.lggr, client, privKey.PublicKey) + channelDefinitionCache := datastreams.NewChannelDefinitionCache() + + return NewStreamsProvider(configWatcher, transmitter, r.lggr, channelDefinitionCache), nil +} + func (r *Relayer) NewFunctionsProvider(rargs commontypes.RelayArgs, pargs commontypes.PluginArgs) (commontypes.FunctionsProvider, error) { lggr := r.lggr.Named("FunctionsProvider").Named(rargs.ExternalJobID.String()) // TODO(FUN-668): Not ready yet (doesn't implement FunctionsEvents() properly) diff --git a/core/services/relay/evm/functions/logpoller_wrapper.go b/core/services/relay/evm/functions/logpoller_wrapper.go index e7f3a1a96af..fa85e645da5 100644 --- a/core/services/relay/evm/functions/logpoller_wrapper.go +++ b/core/services/relay/evm/functions/logpoller_wrapper.go @@ -327,7 +327,7 @@ func (l *logPollerWrapper) SubscribeToUpdates(subscriberName string, subscriber if l.pluginConfig.ContractVersion == 0 { // in V0, immediately set contract address to Oracle contract and never update again if err := subscriber.UpdateRoutes(l.routerContract.Address(), l.routerContract.Address()); err != nil { - l.lggr.Errorw("LogPollerWrapper: Failed to update routes", "subscriberName", subscriberName, "error", err) + l.lggr.Errorw("LogPollerWrapper: Failed to update routes", "subscriberName", subscriberName, "err", err) } } else if l.pluginConfig.ContractVersion == 1 { l.mu.Lock() @@ -422,7 +422,7 @@ func (l *logPollerWrapper) handleRouteUpdate(activeCoordinator common.Address, p for _, subscriber := range l.subscribers { err := subscriber.UpdateRoutes(activeCoordinator, proposedCoordinator) if err != nil { - l.lggr.Errorw("LogPollerWrapper: Failed to update routes", "error", err) + l.lggr.Errorw("LogPollerWrapper: Failed to update routes", "err", err) } } } diff --git a/core/services/relay/evm/mercury/transmitter.go b/core/services/relay/evm/mercury/transmitter.go index 40a51b9d92d..7244144af81 100644 --- a/core/services/relay/evm/mercury/transmitter.go +++ b/core/services/relay/evm/mercury/transmitter.go @@ -107,7 +107,6 @@ type mercuryTransmitter struct { services.StateMachine lggr logger.Logger rpcClient wsrpc.Client - cfgTracker ConfigTracker persistenceManager *PersistenceManager codec TransmitterReportDecoder @@ -148,14 +147,13 @@ func getPayloadTypes() abi.Arguments { }) } -func NewTransmitter(lggr logger.Logger, cfgTracker ConfigTracker, rpcClient wsrpc.Client, fromAccount ed25519.PublicKey, jobID int32, feedID [32]byte, db *sqlx.DB, cfg pg.QConfig, codec TransmitterReportDecoder) *mercuryTransmitter { +func NewTransmitter(lggr logger.Logger, rpcClient wsrpc.Client, fromAccount ed25519.PublicKey, jobID int32, feedID [32]byte, db *sqlx.DB, cfg pg.QConfig, codec TransmitterReportDecoder) *mercuryTransmitter { feedIDHex := fmt.Sprintf("0x%x", feedID[:]) persistenceManager := NewPersistenceManager(lggr, NewORM(db, lggr, cfg), jobID, maxTransmitQueueSize, flushDeletesFrequency, pruneFrequency) return &mercuryTransmitter{ services.StateMachine{}, lggr.Named("MercuryTransmitter").With("feedID", feedIDHex), rpcClient, - cfgTracker, persistenceManager, codec, feedID, @@ -241,7 +239,7 @@ func (mt *mercuryTransmitter) runDeleteQueueLoop() { case req := <-mt.deleteQueue: for { if err := mt.persistenceManager.Delete(runloopCtx, req); err != nil { - mt.lggr.Errorw("Failed to delete transmit request record", "error", err, "req", req) + mt.lggr.Errorw("Failed to delete transmit request record", "err", err, "req", req) mt.transmitQueueDeleteErrorCount.Inc() select { case <-time.After(b.Duration()): diff --git a/core/services/relay/evm/mercury/transmitter_test.go b/core/services/relay/evm/mercury/transmitter_test.go index c8a68d41a16..942a89b291d 100644 --- a/core/services/relay/evm/mercury/transmitter_test.go +++ b/core/services/relay/evm/mercury/transmitter_test.go @@ -27,6 +27,7 @@ func Test_MercuryTransmitter_Transmit(t *testing.T) { pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) q := NewTransmitQueue(lggr, "", 0, nil, nil) + codec := new(mockCodec) t.Run("v1 report transmission successfully enqueued", func(t *testing.T) { report := sampleV1Report @@ -40,7 +41,8 @@ func Test_MercuryTransmitter_Transmit(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), nil) + // func NewTransmitter(lggr logger.Logger, rpcClient wsrpc.Client, fromAccount ed25519.PublicKey, jobID int32, feedID [32]byte, db *sqlx.DB, cfg pg.QConfig, codec TransmitterReportDecoder) *mercuryTransmitter { + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) mt.queue = q err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) @@ -58,7 +60,7 @@ func Test_MercuryTransmitter_Transmit(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) mt.queue = q err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) @@ -76,7 +78,7 @@ func Test_MercuryTransmitter_Transmit(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) mt.queue = q err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) @@ -88,6 +90,8 @@ func Test_MercuryTransmitter_LatestTimestamp(t *testing.T) { t.Parallel() lggr := logger.TestLogger(t) db := pgtest.NewSqlxDB(t) + var jobID int32 + codec := new(mockCodec) t.Run("successful query", func(t *testing.T) { c := mocks.MockWSRPCClient{ @@ -101,7 +105,7 @@ func Test_MercuryTransmitter_LatestTimestamp(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) ts, err := mt.LatestTimestamp(testutils.Context(t)) require.NoError(t, err) @@ -116,7 +120,7 @@ func Test_MercuryTransmitter_LatestTimestamp(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) ts, err := mt.LatestTimestamp(testutils.Context(t)) require.NoError(t, err) @@ -129,7 +133,7 @@ func Test_MercuryTransmitter_LatestTimestamp(t *testing.T) { return nil, errors.New("something exploded") }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) _, err := mt.LatestTimestamp(testutils.Context(t)) require.Error(t, err) assert.Contains(t, err.Error(), "something exploded") @@ -151,6 +155,7 @@ func Test_MercuryTransmitter_LatestPrice(t *testing.T) { t.Parallel() lggr := logger.TestLogger(t) db := pgtest.NewSqlxDB(t) + var jobID int32 codec := new(mockCodec) @@ -167,7 +172,7 @@ func Test_MercuryTransmitter_LatestPrice(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), codec) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) t.Run("BenchmarkPriceFromReport succeeds", func(t *testing.T) { codec.val = originalPrice @@ -197,7 +202,7 @@ func Test_MercuryTransmitter_LatestPrice(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) price, err := mt.LatestPrice(testutils.Context(t), sampleFeedID) require.NoError(t, err) @@ -210,7 +215,7 @@ func Test_MercuryTransmitter_LatestPrice(t *testing.T) { return nil, errors.New("something exploded") }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) _, err := mt.LatestPrice(testutils.Context(t), sampleFeedID) require.Error(t, err) assert.Contains(t, err.Error(), "something exploded") @@ -222,6 +227,8 @@ func Test_MercuryTransmitter_FetchInitialMaxFinalizedBlockNumber(t *testing.T) { lggr := logger.TestLogger(t) db := pgtest.NewSqlxDB(t) + var jobID int32 + codec := new(mockCodec) t.Run("successful query", func(t *testing.T) { c := mocks.MockWSRPCClient{ @@ -235,7 +242,7 @@ func Test_MercuryTransmitter_FetchInitialMaxFinalizedBlockNumber(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) bn, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) require.NoError(t, err) @@ -250,7 +257,7 @@ func Test_MercuryTransmitter_FetchInitialMaxFinalizedBlockNumber(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, jobID, sampleFeedID, db, pgtest.NewQConfig(true), codec) bn, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) require.NoError(t, err) @@ -262,7 +269,7 @@ func Test_MercuryTransmitter_FetchInitialMaxFinalizedBlockNumber(t *testing.T) { return nil, errors.New("something exploded") }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), codec) _, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) require.Error(t, err) assert.Contains(t, err.Error(), "something exploded") @@ -279,7 +286,7 @@ func Test_MercuryTransmitter_FetchInitialMaxFinalizedBlockNumber(t *testing.T) { return out, nil }, } - mt := NewTransmitter(lggr, nil, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), nil) + mt := NewTransmitter(lggr, c, sampleClientPubKey, 0, sampleFeedID, db, pgtest.NewQConfig(true), codec) _, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) require.Error(t, err) assert.Contains(t, err.Error(), "latestReport failed; mismatched feed IDs, expected: 0x1c916b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472, got: 0x") diff --git a/core/services/relay/evm/mercury/v1/data_source.go b/core/services/relay/evm/mercury/v1/data_source.go index ce48ec6cf94..7f41bd1e36c 100644 --- a/core/services/relay/evm/mercury/v1/data_source.go +++ b/core/services/relay/evm/mercury/v1/data_source.go @@ -295,7 +295,7 @@ func (ds *datasource) setLatestBlocks(ctx context.Context, obs *v1types.Observat latestBlocks, err := ds.mercuryChainReader.LatestHeads(ctx, nBlocksObservation) if err != nil { - ds.lggr.Errorw("failed to read latest blocks", "error", err) + ds.lggr.Errorw("failed to read latest blocks", "err", err) return err } diff --git a/core/services/relay/evm/mocks/loop_relay_adapter.go b/core/services/relay/evm/mocks/loop_relay_adapter.go index 5b927f1b8ac..b196e10da4d 100644 --- a/core/services/relay/evm/mocks/loop_relay_adapter.go +++ b/core/services/relay/evm/mocks/loop_relay_adapter.go @@ -226,6 +226,32 @@ func (_m *LoopRelayAdapter) NewPluginProvider(_a0 context.Context, _a1 types.Rel return r0, r1 } +// NewStreamsProvider provides a mock function with given fields: _a0, _a1, _a2 +func (_m *LoopRelayAdapter) NewStreamsProvider(_a0 context.Context, _a1 types.RelayArgs, _a2 types.PluginArgs) (types.StreamsProvider, error) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 types.StreamsProvider + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, types.RelayArgs, types.PluginArgs) (types.StreamsProvider, error)); ok { + return rf(_a0, _a1, _a2) + } + if rf, ok := ret.Get(0).(func(context.Context, types.RelayArgs, types.PluginArgs) types.StreamsProvider); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(types.StreamsProvider) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, types.RelayArgs, types.PluginArgs) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Ready provides a mock function with given fields: func (_m *LoopRelayAdapter) Ready() error { ret := _m.Called() diff --git a/core/services/relay/evm/streams_provider.go b/core/services/relay/evm/streams_provider.go new file mode 100644 index 00000000000..c576718748e --- /dev/null +++ b/core/services/relay/evm/streams_provider.go @@ -0,0 +1,86 @@ +package evm + +import ( + "context" + "errors" + + "github.com/smartcontractkit/chainlink-common/pkg/services" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + relaytypes "github.com/smartcontractkit/chainlink-common/pkg/types" + datastreams "github.com/smartcontractkit/chainlink-data-streams/streams" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/streams" +) + +var _ commontypes.StreamsProvider = (*streamsProvider)(nil) + +type streamsProvider struct { + configWatcher *configWatcher + transmitter streams.Transmitter + logger logger.Logger + channelDefinitionCache datastreams.ChannelDefinitionCache + + ms services.MultiStart +} + +func NewStreamsProvider( + configWatcher *configWatcher, + transmitter streams.Transmitter, + lggr logger.Logger, + channelDefinitionCache datastreams.ChannelDefinitionCache, +) relaytypes.StreamsProvider { + return &streamsProvider{ + configWatcher, + transmitter, + lggr, + channelDefinitionCache, + services.MultiStart{}, + } +} + +func (p *streamsProvider) Start(ctx context.Context) error { + return p.ms.Start(ctx, p.configWatcher, p.transmitter, p.channelDefinitionCache) +} + +func (p *streamsProvider) Close() error { + return p.ms.Close() +} + +func (p *streamsProvider) Ready() error { + return errors.Join(p.configWatcher.Ready(), p.transmitter.Ready(), p.channelDefinitionCache.Ready()) +} + +func (p *streamsProvider) Name() string { + return p.logger.Name() +} + +func (p *streamsProvider) HealthReport() map[string]error { + report := map[string]error{} + services.CopyHealth(report, p.configWatcher.HealthReport()) + services.CopyHealth(report, p.transmitter.HealthReport()) + services.CopyHealth(report, p.channelDefinitionCache.HealthReport()) + return report +} + +func (p *streamsProvider) ContractConfigTracker() ocrtypes.ContractConfigTracker { + return p.configWatcher.ContractConfigTracker() +} + +func (p *streamsProvider) OffchainConfigDigester() ocrtypes.OffchainConfigDigester { + return p.configWatcher.OffchainConfigDigester() +} + +func (p *streamsProvider) OnchainConfigCodec() datastreams.OnchainConfigCodec { + // TODO: This should probably be moved to core since its chain-specific + return &datastreams.JSONOnchainConfigCodec{} +} + +func (p *streamsProvider) ContractTransmitter() commontypes.StreamsTransmitter { + return p.transmitter +} + +func (p *streamsProvider) ChannelDefinitionCache() datastreams.ChannelDefinitionCache { + return p.channelDefinitionCache +} diff --git a/core/services/streams/data_source.go b/core/services/streams/data_source.go new file mode 100644 index 00000000000..dfc3afea727 --- /dev/null +++ b/core/services/streams/data_source.go @@ -0,0 +1,91 @@ +package streams + +// TODO: llo datasource +import ( + "context" + "fmt" + "math/big" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-data-streams/streams" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +var ( + promMissingStreamCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_stream_missing_count", + Help: "Number of times we tried to observe a stream, but it was missing", + }, + []string{"streamID"}, + ) + promObservationErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "llo_stream_observation_error_count", + Help: "Number of times we tried to observe a stream, but it failed with an error", + }, + []string{"streamID"}, + ) +) + +type ErrMissingStream struct { + id string +} + +func (e ErrMissingStream) Error() string { + return fmt.Sprintf("missing stream definition for: %q", e.id) +} + +var _ streams.DataSource = &dataSource{} + +type dataSource struct { + lggr logger.Logger + streamCache StreamCache +} + +func NewDataSource(lggr logger.Logger, streamCache StreamCache) streams.DataSource { + // TODO: lggr should include job ID + return &dataSource{lggr, streamCache} +} + +func (d *dataSource) Observe(ctx context.Context, streamIDs map[streams.StreamID]struct{}) (streams.StreamValues, error) { + // There is no "observationSource" (AKA pipeline) + // Need a concept of "streams" + // Streams are referenced by ID from the on-chain config + // Each stream contains its own pipeline + // See: https://docs.google.com/document/d/1l1IiDOL1QSteLTnhmiGnJAi6QpcSpyOe0nkqS7D3SvU/edit for stream ID naming + + var wg sync.WaitGroup + wg.Add(len(streamIDs)) + sv := make(streams.StreamValues) + var mu sync.Mutex + + for streamID := range streamIDs { + go func(streamID streams.StreamID) { + defer wg.Done() + + var res streams.ObsResult[*big.Int] + + stream, exists := d.streamCache.Get(streamID) + if exists { + res.Val, res.Err = stream.Observe(ctx) + if res.Err != nil { + d.lggr.Debugw("Observation failed for stream", "err", res.Err, "streamID", streamID) + promObservationErrorCount.WithLabelValues(streamID.String()).Inc() + } + } else { + d.lggr.Errorw(fmt.Sprintf("Missing stream: %q", streamID), "streamID", streamID) + promMissingStreamCount.WithLabelValues(streamID.String()).Inc() + res.Err = ErrMissingStream{streamID.String()} + } + + mu.Lock() + defer mu.Unlock() + sv[streamID] = res + }(streamID) + } + + wg.Wait() + + return sv, nil +} diff --git a/core/services/streams/data_source_test.go b/core/services/streams/data_source_test.go new file mode 100644 index 00000000000..6b78eba3d8a --- /dev/null +++ b/core/services/streams/data_source_test.go @@ -0,0 +1,90 @@ +package streams + +import ( + "context" + "errors" + "math/big" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +type mockStream struct { + dp DataPoint + err error +} + +func (m *mockStream) Observe(ctx context.Context) (DataPoint, error) { + return m.dp, m.err +} + +func Test_DataSource(t *testing.T) { + lggr := logger.TestLogger(t) + sc := newStreamCache(nil) + ds := NewDataSource(lggr, sc) + ctx := testutils.Context(t) + + streamIDs := make(map[streams.StreamID]struct{}) + streamIDs[streams.StreamID("ETH/USD")] = struct{}{} + streamIDs[streams.StreamID("BTC/USD")] = struct{}{} + streamIDs[streams.StreamID("LINK/USD")] = struct{}{} + + t.Run("Observe", func(t *testing.T) { + t.Run("returns errors if no streams are defined", func(t *testing.T) { + vals, err := ds.Observe(ctx, streamIDs) + assert.NoError(t, err) + + assert.Equal(t, streams.StreamValues{ + "BTC/USD": streams.ObsResult[*big.Int]{Val: nil, Err: ErrMissingStream{id: "BTC/USD"}}, + "ETH/USD": streams.ObsResult[*big.Int]{Val: nil, Err: ErrMissingStream{id: "ETH/USD"}}, + "LINK/USD": streams.ObsResult[*big.Int]{Val: nil, Err: ErrMissingStream{id: "LINK/USD"}}, + }, vals) + }) + t.Run("observes each stream with success and returns values matching map argument", func(t *testing.T) { + sc.streams["ETH/USD"] = &mockStream{ + dp: big.NewInt(2181), + } + sc.streams["BTC/USD"] = &mockStream{ + dp: big.NewInt(40602), + } + sc.streams["LINK/USD"] = &mockStream{ + dp: big.NewInt(15), + } + + vals, err := ds.Observe(ctx, streamIDs) + assert.NoError(t, err) + + assert.Equal(t, streams.StreamValues{ + "BTC/USD": streams.ObsResult[*big.Int]{Val: big.NewInt(40602), Err: nil}, + "ETH/USD": streams.ObsResult[*big.Int]{Val: big.NewInt(2181), Err: nil}, + "LINK/USD": streams.ObsResult[*big.Int]{Val: big.NewInt(15), Err: nil}, + }, vals) + }) + t.Run("observes each stream and returns success/errors", func(t *testing.T) { + sc.streams["ETH/USD"] = &mockStream{ + dp: big.NewInt(2181), + err: errors.New("something exploded"), + } + sc.streams["BTC/USD"] = &mockStream{ + dp: big.NewInt(40602), + } + sc.streams["LINK/USD"] = &mockStream{ + err: errors.New("something exploded 2"), + } + + vals, err := ds.Observe(ctx, streamIDs) + assert.NoError(t, err) + + assert.Equal(t, streams.StreamValues{ + "BTC/USD": streams.ObsResult[*big.Int]{Val: big.NewInt(40602), Err: nil}, + "ETH/USD": streams.ObsResult[*big.Int]{Val: big.NewInt(2181), Err: errors.New("something exploded")}, + "LINK/USD": streams.ObsResult[*big.Int]{Val: nil, Err: errors.New("something exploded 2")}, + }, vals) + }) + }) +} diff --git a/core/services/streams/delegate.go b/core/services/streams/delegate.go new file mode 100644 index 00000000000..e09a26672e3 --- /dev/null +++ b/core/services/streams/delegate.go @@ -0,0 +1,90 @@ +package streams + +import ( + "context" + "fmt" + + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-data-streams/streams" + ocrcommontypes "github.com/smartcontractkit/libocr/commontypes" + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2/types" + ocr2plus "github.com/smartcontractkit/libocr/offchainreporting2plus" + ocr3types "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/job" + "github.com/smartcontractkit/chainlink/v2/core/services/pg" +) + +var _ job.ServiceCtx = &delegate{} + +type delegate struct { + cfg DelegateConfig + codecs map[commontypes.StreamsReportFormat]streams.ReportCodec +} + +type DelegateConfig struct { + Logger logger.Logger + Queryer pg.Queryer + Runner Runner + + // OCR3 + BinaryNetworkEndpointFactory ocr2types.BinaryNetworkEndpointFactory + V2Bootstrappers []ocrcommontypes.BootstrapperLocator + ContractConfigTracker ocr2types.ContractConfigTracker + ContractTransmitter ocr3types.ContractTransmitter[commontypes.StreamsReportInfo] + Database ocr3types.Database + OCRLogger ocrcommontypes.Logger + MonitoringEndpoint ocrcommontypes.MonitoringEndpoint + OffchainConfigDigester ocr2types.OffchainConfigDigester + OffchainKeyring ocr2types.OffchainKeyring + OnchainKeyring ocr3types.OnchainKeyring[commontypes.StreamsReportInfo] + LocalConfig ocr2types.LocalConfig +} + +func NewDelegate(cfg DelegateConfig) job.ServiceCtx { + // TODO: add the chain codecs here + // TODO: nil checks? + codecs := make(map[commontypes.StreamsReportFormat]streams.ReportCodec) + return &delegate{cfg, codecs} +} + +func (d *delegate) Start(ctx context.Context) error { + // create the oracle from config values + // TODO: Do these services need starting? + prrc := streams.NewPredecessorRetirementReportCache() + src := streams.NewShouldRetireCache() + cdc := streams.NewChannelDefinitionCache() + orm := NewORM(d.cfg.Queryer) + sc := NewStreamCache(orm) + if err := sc.Load(ctx, d.cfg.Logger.Named("StreamCache"), d.cfg.Runner); err != nil { + return err + } + ds := NewDataSource(d.cfg.Logger.Named("DataSource"), sc) + llo, err := ocr2plus.NewOracle(ocr2plus.OCR3OracleArgs[commontypes.StreamsReportInfo]{ + BinaryNetworkEndpointFactory: d.cfg.BinaryNetworkEndpointFactory, + V2Bootstrappers: d.cfg.V2Bootstrappers, + ContractConfigTracker: d.cfg.ContractConfigTracker, + ContractTransmitter: d.cfg.ContractTransmitter, + Database: d.cfg.Database, + LocalConfig: d.cfg.LocalConfig, + Logger: d.cfg.OCRLogger, + MonitoringEndpoint: d.cfg.MonitoringEndpoint, + OffchainConfigDigester: d.cfg.OffchainConfigDigester, + OffchainKeyring: d.cfg.OffchainKeyring, + OnchainKeyring: d.cfg.OnchainKeyring, + ReportingPluginFactory: streams.NewPluginFactory( + prrc, src, cdc, ds, d.cfg.Logger.Named("StreamsReportingPlugin"), d.codecs, + ), + }) + + if err != nil { + return fmt.Errorf("%w: failed to create new OCR oracle", err) + } + + return llo.Start() +} + +func (d *delegate) Close() error { + panic("TODO") +} diff --git a/core/services/streams/keyring.go b/core/services/streams/keyring.go new file mode 100644 index 00000000000..3faab4be115 --- /dev/null +++ b/core/services/streams/keyring.go @@ -0,0 +1,60 @@ +package streams + +import ( + "fmt" + + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/chainlink-data-streams/streams" + "github.com/smartcontractkit/libocr/offchainreporting2/types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" +) + +type StreamsOnchainKeyring ocr3types.OnchainKeyring[commontypes.StreamsReportInfo] + +var _ StreamsOnchainKeyring = &onchainKeyring{} + +type Key interface { + Sign3(digest ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report) (signature []byte, err error) + Verify3(publicKey ocrtypes.OnchainPublicKey, cd ocrtypes.ConfigDigest, seqNr uint64, r ocrtypes.Report, signature []byte) bool + PublicKey() ocrtypes.OnchainPublicKey + MaxSignatureLength() int +} + +type onchainKeyring struct { + evm Key +} + +func NewOnchainKeyring(evm Key) StreamsOnchainKeyring { + return &onchainKeyring{ + evm: evm, + } +} + +func (okr *onchainKeyring) PublicKey() types.OnchainPublicKey { + // TODO: Combine this in some way for multiple chains + return okr.evm.PublicKey() +} + +func (okr *onchainKeyring) MaxSignatureLength() int { + // TODO: Needs to be max of all chain sigs + return okr.evm.MaxSignatureLength() +} + +func (okr *onchainKeyring) Sign(digest types.ConfigDigest, seqNr uint64, r ocr3types.ReportWithInfo[commontypes.StreamsReportInfo]) (signature []byte, err error) { + switch r.Info.ReportFormat { + case streams.ReportFormatEVM: + return okr.evm.Sign3(digest, seqNr, r.Report) + default: + return nil, fmt.Errorf("unsupported format: %q", r.Info.ReportFormat) + } +} + +func (okr *onchainKeyring) Verify(key types.OnchainPublicKey, digest types.ConfigDigest, seqNr uint64, r ocr3types.ReportWithInfo[commontypes.StreamsReportInfo], signature []byte) bool { + switch r.Info.ReportFormat { + case streams.ReportFormatEVM: + return okr.evm.Verify3(key, digest, seqNr, r.Report, signature) + default: + return false + } +} diff --git a/core/services/streams/keyring_test.go b/core/services/streams/keyring_test.go new file mode 100644 index 00000000000..b958164bdec --- /dev/null +++ b/core/services/streams/keyring_test.go @@ -0,0 +1,7 @@ +package streams + +import "testing" + +func Test_Keyring(t *testing.T) { + t.Fatal("TODO") +} diff --git a/core/services/streams/orm.go b/core/services/streams/orm.go new file mode 100644 index 00000000000..d2cef471468 --- /dev/null +++ b/core/services/streams/orm.go @@ -0,0 +1,44 @@ +package streams + +import ( + "context" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/pg" +) + +type ORM interface { + StreamCacheORM +} + +var _ ORM = &orm{} + +type orm struct { + q pg.Queryer +} + +func NewORM(q pg.Queryer) ORM { + return &orm{q} +} + +func (o *orm) LoadStreams(ctx context.Context, lggr logger.Logger, runner Runner, m map[streams.StreamID]Stream) error { + rows, err := o.q.QueryContext(ctx, "SELECT s.id, ps.id, ps.dot_dag_source, ps.max_task_duration FROM streams s JOIN pipeline_specs ps ON ps.id = s.pipeline_spec_id") + if err != nil { + // TODO: retries? + return err + } + + for rows.Next() { + var strm stream + if err := rows.Scan(&strm.id, &strm.spec.ID, &strm.spec.DotDagSource, &strm.spec.MaxTaskDuration); err != nil { + return err + } + strm.lggr = lggr.Named("Stream").With("streamID", strm.id) + strm.runner = runner + + m[strm.id] = &strm + } + return rows.Err() +} diff --git a/core/services/streams/orm_test.go b/core/services/streams/orm_test.go new file mode 100644 index 00000000000..1c72e0347e9 --- /dev/null +++ b/core/services/streams/orm_test.go @@ -0,0 +1,71 @@ +package streams + +import ( + "context" + "testing" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" + + "github.com/stretchr/testify/assert" +) + +type mockRunner struct{} + +func (m *mockRunner) ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars, l logger.Logger) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) { + return +} + +func Test_ORM(t *testing.T) { + db := pgtest.NewSqlxDB(t) + orm := NewORM(db) + ctx := testutils.Context(t) + lggr := logger.TestLogger(t) + runner := &mockRunner{} + + t.Run("LoadStreams", func(t *testing.T) { + t.Run("nothing in database", func(t *testing.T) { + m := make(map[streams.StreamID]Stream) + err := orm.LoadStreams(ctx, lggr, runner, m) + assert.NoError(t, err) + + assert.Len(t, m, 0) + }) + t.Run("loads streams from database", func(t *testing.T) { + pgtest.MustExec(t, db, ` +WITH pipeline_specs AS ( + INSERT INTO pipeline_specs (dot_dag_source, created_at) VALUES + ('foo', NOW()), + ('bar', NOW()), + ('baz', NOW()) + RETURNING id, dot_dag_source +) +INSERT INTO streams(id, pipeline_spec_id, created_at) +SELECT CONCAT('stream-', pipeline_specs.dot_dag_source), pipeline_specs.id, NOW() +FROM pipeline_specs +`) + + m := make(map[streams.StreamID]Stream) + err := orm.LoadStreams(ctx, lggr, runner, m) + assert.NoError(t, err) + + assert.Len(t, m, 3) + assert.Contains(t, m, streams.StreamID("stream-foo")) + assert.Contains(t, m, streams.StreamID("stream-bar")) + assert.Contains(t, m, streams.StreamID("stream-baz")) + + // test one of the streams to ensure it got loaded correctly + s := m["stream-foo"].(*stream) + assert.Equal(t, streams.StreamID("stream-foo"), s.id) + assert.NotNil(t, s.lggr) + assert.Equal(t, "foo", s.spec.DotDagSource) + assert.NotZero(t, s.spec.ID) + assert.NotNil(t, s.runner) + assert.Equal(t, runner, s.runner) + }) + }) +} diff --git a/core/services/streams/stream.go b/core/services/streams/stream.go new file mode 100644 index 00000000000..e0a004b1d8a --- /dev/null +++ b/core/services/streams/stream.go @@ -0,0 +1,118 @@ +package streams + +import ( + "context" + "fmt" + "math/big" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" + "github.com/smartcontractkit/chainlink/v2/core/utils" +) + +type Runner interface { + ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars, l logger.Logger) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) +} + +// TODO: Generalize to beyond simply an int +type DataPoint *big.Int + +type Stream interface { + Observe(ctx context.Context) (DataPoint, error) +} + +type stream struct { + id streams.StreamID + lggr logger.Logger + spec pipeline.Spec + runner Runner +} + +func NewStream(lggr logger.Logger, id streams.StreamID, spec pipeline.Spec, runner Runner) Stream { + return newStream(lggr, id, spec, runner) +} + +func newStream(lggr logger.Logger, id streams.StreamID, spec pipeline.Spec, runner Runner) *stream { + return &stream{id, lggr, spec, runner} +} + +func (s *stream) Observe(ctx context.Context) (DataPoint, error) { + var run *pipeline.Run + run, trrs, err := s.executeRun(ctx) + if err != nil { + return nil, fmt.Errorf("Observe failed while executing run: %w", err) + } + s.lggr.Tracew("Observe executed run", "run", run) + // FIXME: runResults?? + // select { + // case s.runResults <- run: + // default: + // s.lggr.Warnf("unable to enqueue run save for job ID %d, buffer full", s.spec.JobID) + // } + + // NOTE: trrs comes back as _all_ tasks, but we only want the terminal ones + // They are guaranteed to be sorted by index asc so should be in the correct order + var finaltrrs []pipeline.TaskRunResult + for _, trr := range trrs { + if trr.IsTerminal() { + finaltrrs = append(finaltrrs, trr) + } + } + + // FIXME: How to handle arbitrary-shaped inputs? + // For now just assume everything is one *big.Int + parsed, err := s.parse(finaltrrs) + if err != nil { + return nil, fmt.Errorf("Observe failed while parsing run results: %w", err) + } + return parsed, nil + +} + +// The context passed in here has a timeout of (ObservationTimeout + ObservationGracePeriod). +// Upon context cancellation, its expected that we return any usable values within ObservationGracePeriod. +func (s *stream) executeRun(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { + // TODO: does it need some kind of debugging stuff here? + vars := pipeline.NewVarsFrom(map[string]interface{}{}) + + run, trrs, err := s.runner.ExecuteRun(ctx, s.spec, vars, s.lggr) + if err != nil { + return nil, nil, fmt.Errorf("error executing run for spec ID %v: %w", s.spec.ID, err) + } + + return run, trrs, err +} + +// returns error on parse errors: if something is the wrong type +func (s *stream) parse(trrs pipeline.TaskRunResults) (*big.Int, error) { + var finaltrrs []pipeline.TaskRunResult + for _, trr := range trrs { + // only return terminal trrs from executeRun + if trr.IsTerminal() { + finaltrrs = append(finaltrrs, trr) + } + } + + // pipeline.TaskRunResults comes ordered asc by index, this is guaranteed + // by the pipeline executor + if len(finaltrrs) != 1 { + return nil, fmt.Errorf("invalid number of results, expected: 1, got: %d", len(finaltrrs)) + } + res := finaltrrs[0].Result + if res.Error != nil { + return nil, res.Error + } else if val, err := toBigInt(res.Value); err != nil { + return nil, fmt.Errorf("failed to parse BenchmarkPrice: %w", err) + } else { + return val, nil + } +} + +func toBigInt(val interface{}) (*big.Int, error) { + dec, err := utils.ToDecimal(val) + if err != nil { + return nil, err + } + return dec.BigInt(), nil +} diff --git a/core/services/streams/stream_cache.go b/core/services/streams/stream_cache.go new file mode 100644 index 00000000000..1c322cb5de1 --- /dev/null +++ b/core/services/streams/stream_cache.go @@ -0,0 +1,43 @@ +package streams + +import ( + "context" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +type StreamCacheORM interface { + LoadStreams(ctx context.Context, lggr logger.Logger, runner Runner, m map[streams.StreamID]Stream) error +} + +type StreamCache interface { + Get(streamID streams.StreamID) (Stream, bool) + Load(ctx context.Context, lggr logger.Logger, runner Runner) error +} + +type streamCache struct { + orm StreamCacheORM + streams map[streams.StreamID]Stream +} + +func NewStreamCache(orm StreamCacheORM) StreamCache { + return newStreamCache(orm) +} + +func newStreamCache(orm StreamCacheORM) *streamCache { + return &streamCache{ + orm, + make(map[streams.StreamID]Stream), + } +} + +func (s *streamCache) Get(streamID streams.StreamID) (Stream, bool) { + strm, exists := s.streams[streamID] + return strm, exists +} + +func (s *streamCache) Load(ctx context.Context, lggr logger.Logger, runner Runner) error { + return s.orm.LoadStreams(ctx, lggr, runner, s.streams) +} diff --git a/core/services/streams/stream_cache_test.go b/core/services/streams/stream_cache_test.go new file mode 100644 index 00000000000..902e610a68f --- /dev/null +++ b/core/services/streams/stream_cache_test.go @@ -0,0 +1,73 @@ +package streams + +import ( + "context" + "errors" + "maps" + "math/big" + "testing" + + "github.com/smartcontractkit/chainlink-data-streams/streams" + + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/logger" + + "github.com/stretchr/testify/assert" +) + +type mockORM struct { + m map[streams.StreamID]Stream + err error +} + +func (orm *mockORM) LoadStreams(ctx context.Context, lggr logger.Logger, runner Runner, m map[streams.StreamID]Stream) error { + maps.Copy(m, orm.m) + return orm.err +} + +func Test_StreamCache(t *testing.T) { + t.Run("Load", func(t *testing.T) { + orm := &mockORM{} + sc := newStreamCache(orm) + lggr := logger.TestLogger(t) + runner := &mockRunner{} + + t.Run("populates cache from database using ORM", func(t *testing.T) { + assert.Len(t, sc.streams, 0) + err := sc.Load(testutils.Context(t), lggr, runner) + assert.NoError(t, err) + assert.Len(t, sc.streams, 0) + + v, exists := sc.Get("foo") + assert.Nil(t, v) + assert.False(t, exists) + + orm.m = make(map[streams.StreamID]Stream) + orm.m["foo"] = &mockStream{dp: big.NewInt(1)} + orm.m["bar"] = &mockStream{dp: big.NewInt(2)} + orm.m["baz"] = &mockStream{dp: big.NewInt(3)} + + err = sc.Load(testutils.Context(t), lggr, runner) + assert.NoError(t, err) + assert.Len(t, sc.streams, 3) + + v, exists = sc.Get("foo") + assert.True(t, exists) + assert.Equal(t, orm.m["foo"], v) + + v, exists = sc.Get("bar") + assert.True(t, exists) + assert.Equal(t, orm.m["bar"], v) + + v, exists = sc.Get("baz") + assert.True(t, exists) + assert.Equal(t, orm.m["baz"], v) + }) + + t.Run("returns error if db errors", func(t *testing.T) { + orm.err = errors.New("something exploded") + err := sc.Load(testutils.Context(t), lggr, runner) + assert.EqualError(t, err, "something exploded") + }) + }) +} diff --git a/core/services/streams/transmitter.go b/core/services/streams/transmitter.go new file mode 100644 index 00000000000..db69e06d0f3 --- /dev/null +++ b/core/services/streams/transmitter.go @@ -0,0 +1,74 @@ +package streams + +// TODO: llo transmitter + +import ( + "context" + "crypto/ed25519" + "fmt" + + "github.com/smartcontractkit/chainlink-common/pkg/services" + commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" + "github.com/smartcontractkit/libocr/offchainreporting2/types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" +) + +type Transmitter interface { + commontypes.StreamsTransmitter + services.Service +} + +type transmitter struct { + services.StateMachine + lggr logger.Logger + rpcClient wsrpc.Client + fromAccount string +} + +func NewTransmitter(lggr logger.Logger, rpcClient wsrpc.Client, fromAccount ed25519.PublicKey) Transmitter { + return &transmitter{ + services.StateMachine{}, + lggr, + rpcClient, + fmt.Sprintf("%x", fromAccount), + } +} + +func (t *transmitter) Start(ctx context.Context) error { + // TODO + return nil +} + +func (t *transmitter) Close() error { + // TODO + return nil +} + +func (t *transmitter) HealthReport() map[string]error { + report := map[string]error{t.Name(): t.Healthy()} + services.CopyHealth(report, t.rpcClient.HealthReport()) + // FIXME + // services.CopyHealth(report, t.queue.HealthReport()) + return report +} + +func (t *transmitter) Name() string { return t.lggr.Name() } + +func (t *transmitter) Transmit( + context.Context, + types.ConfigDigest, + uint64, + ocr3types.ReportWithInfo[commontypes.StreamsReportInfo], + []types.AttributedOnchainSignature, +) error { + panic("TODO") +} + +// FromAccount returns the stringified (hex) CSA public key +func (t *transmitter) FromAccount() (ocrtypes.Account, error) { + return ocrtypes.Account(t.fromAccount), nil +} diff --git a/core/services/vrf/v1/listener_v1.go b/core/services/vrf/v1/listener_v1.go index 66a8ddcd58c..238a1c0055d 100644 --- a/core/services/vrf/v1/listener_v1.go +++ b/core/services/vrf/v1/listener_v1.go @@ -368,7 +368,7 @@ func (lsn *Listener) handleLog(lb log.Broadcast, minConfs uint32) { func (lsn *Listener) shouldProcessLog(lb log.Broadcast) bool { consumed, err := lsn.Chain.LogBroadcaster().WasAlreadyConsumed(lb) if err != nil { - lsn.L.Errorw("Could not determine if log was already consumed", "error", err, "txHash", lb.RawLog().TxHash) + lsn.L.Errorw("Could not determine if log was already consumed", "err", err, "txHash", lb.RawLog().TxHash) // Do not process, let lb resend it as a retry mechanism. return false } diff --git a/core/store/migrate/migrations/0213_create_streams.sql b/core/store/migrate/migrations/0213_create_streams.sql new file mode 100644 index 00000000000..d72d0c0a1ab --- /dev/null +++ b/core/store/migrate/migrations/0213_create_streams.sql @@ -0,0 +1,10 @@ +-- +goose Up +CREATE TABLE streams ( + id text PRIMARY KEY, + pipeline_spec_id INT REFERENCES pipeline_specs (id) ON DELETE CASCADE DEFERRABLE INITIALLY IMMEDIATE, + created_at timestamp with time zone NOT NULL +); + + +-- +goose Down +DROP TABLE streams; diff --git a/go.mod b/go.mod index 40359ab16f7..173db46dc69 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( github.com/smartcontractkit/wsrpc v0.7.2 github.com/spf13/cast v1.6.0 github.com/stretchr/testify v1.8.4 + github.com/test-go/testify v1.1.4 github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a github.com/tidwall/gjson v1.17.0 github.com/ugorji/go/codec v1.2.12 @@ -132,6 +133,7 @@ require ( github.com/bgentry/speakeasy v0.1.1-0.20220910012023-760eaf8b6816 // indirect github.com/blendle/zapdriver v1.3.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/btcsuite/btcd/btcutil v1.1.3 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect github.com/bytedance/sonic v1.10.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect @@ -279,6 +281,7 @@ require ( github.com/sethvargo/go-retry v0.2.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/smartcontractkit/chain-selectors v1.0.5 // indirect github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect @@ -348,4 +351,12 @@ replace ( // until merged upstream: https://github.com/mwitkow/grpc-proxy/pull/69 github.com/mwitkow/grpc-proxy => github.com/smartcontractkit/grpc-proxy v0.0.0-20230731113816-f1be6620749f + + // TODO: streams + github.com/smartcontractkit/chainlink-common => /Users/sam/code/smartcontractkit/chainlink-common + github.com/smartcontractkit/chainlink-cosmos => /Users/sam/code/smartcontractkit/chainlink-cosmos + github.com/smartcontractkit/chainlink-data-streams => /Users/sam/code/smartcontractkit/chainlink-data-streams + github.com/smartcontractkit/chainlink-solana => /Users/sam/code/smartcontractkit/chainlink-solana + github.com/smartcontractkit/chainlink-starknet/relayer => /Users/sam/code/smartcontractkit/chainlink-starknet/relayer + ) diff --git a/go.sum b/go.sum index 11b377b11f8..19c48e09198 100644 --- a/go.sum +++ b/go.sum @@ -174,10 +174,12 @@ github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHf github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= -github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= +github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -310,8 +312,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g= github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= @@ -1220,8 +1224,11 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704 h1:T3lFWumvbfM1u/etVq42Afwq/jtNSBSOA8n5jntnNPo= github.com/smartcontractkit/caigo v0.0.0-20230621050857-b29a4ca8c704/go.mod h1:2QuJdEouTWjh5BDy5o/vgGXQtR4Gz8yH1IYB5eT7u4M= +github.com/smartcontractkit/chain-selectors v1.0.5 h1:NOefQsogPZS4aBbWPFrgAyoke0gppN2ojfa8SQkhu8c= +github.com/smartcontractkit/chain-selectors v1.0.5/go.mod h1:WBhLlODF5b95vvx2tdKK55vGACg1+qZpuBhOGu1UXVo= github.com/smartcontractkit/chainlink-automation v1.0.1 h1:vVjBFq2Zsz21kPy1Pb0wpjF9zrbJX+zjXphDeeR4XZk= github.com/smartcontractkit/chainlink-automation v1.0.1/go.mod h1:INSchkV3ntyDdlZKGWA030MPDpp6pbeuiRkRKYFCm2k= +<<<<<<< HEAD github.com/smartcontractkit/chainlink-common v0.1.7-0.20231206181640-faad3f11cfad h1:ysPjfbCPJuVxxFZa1Ifv8OPE20pzvnEHjJrPDUo4gT0= github.com/smartcontractkit/chainlink-common v0.1.7-0.20231206181640-faad3f11cfad/go.mod h1:IdlfCN9rUs8Q/hrOYe8McNBIwEOHEsi0jilb3Cw77xs= github.com/smartcontractkit/chainlink-cosmos v0.4.1-0.20231206164210-03f8b219402e h1:xvqffqFec2HkEcUKrCkm4FDJRnn/+gHmvrE/dz3Zlw8= @@ -1234,6 +1241,10 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231206154215-ec1718b7df3 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20231206154215-ec1718b7df3e/go.mod h1:9YIi413QRRytafTzpWm+Z+5NWBNxSqokhKyeEZ3ynlA= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231205180940-ea2e3e916725 h1:NbhPVwxx+53WN/Uld1V6c4iLgoGvUYFOsVd2kfcexe8= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20231205180940-ea2e3e916725/go.mod h1:vHrPBipRL52NdPp77KXNU2k1IoCUa1B33N9otZQPYko= +======= +github.com/smartcontractkit/chainlink-feeds v0.0.0-20231127231053-2232d3a6766d h1:w4MsbOtNk6nD/mcXLstHWk9hB6g7QLtcAfhPjhwvOaQ= +github.com/smartcontractkit/chainlink-feeds v0.0.0-20231127231053-2232d3a6766d/go.mod h1:YPAfLNowdBwiKiYOwgwtbJHi8AJWbcxkbOY0ItAvkfc= +>>>>>>> 2cf1c5828a (Implement Data Streams plugin) github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs= github.com/smartcontractkit/go-plugin v0.0.0-20231003134350-e49dad63b306 h1:ko88+ZznniNJZbZPWAvHQU8SwKAdHngdDZ+pvVgB5ss= diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 0e29f6f6715..2180e6cc5a6 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1617,8 +1617,15 @@ github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoM github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= +<<<<<<< HEAD github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= +======= +github.com/stretchr/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= +github.com/stretchr/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= +github.com/testcontainers/testcontainers-go v0.23.0 h1:ERYTSikX01QczBLPZpqsETTBO7lInqEP349phDOVJVs= +github.com/testcontainers/testcontainers-go v0.23.0/go.mod h1:3gzuZfb7T9qfcH2pHpV4RLlWrPjeWNQah6XlYQ32c4I= +>>>>>>> 2cf1c5828a (Implement Data Streams plugin) github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a h1:YuO+afVc3eqrjiCUizNCxI53bl/BnPiVwXqLzqYTqgU= github.com/theodesp/go-heaps v0.0.0-20190520121037-88e35354fe0a/go.mod h1:/sfW47zCZp9FrtGcWyo1VjbgDaodxX9ovZvgLb/MxaA= github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e h1:BuzhfgfWQbX0dWzYzT1zsORLnHRv3bcRcsaUk0VmXA8=