From ed7d315f09261096ecf6c1adb6434257b09a21cf Mon Sep 17 00:00:00 2001 From: Matt Luongo Date: Mon, 20 Dec 2021 22:35:30 -0500 Subject: [PATCH 1/4] Refactor blocks in Redux to a networks slice Storing them in the accounts slice never made sense. The status of a block on a particular network isn't account specific. The push for multi-network support means more state will need to be keyed by network. While we only support EVM-based chains, that means chain ID. More refactors will inevitably follow; for now, store the EVM network state in an EVM-specific area with chain ids under it. --- background/main.ts | 52 ++++++++++++++++++- background/redux-slices/accounts.ts | 10 +--- background/redux-slices/index.ts | 2 + background/redux-slices/networks.ts | 48 +++++++++++++++++ .../selectors/activitiesSelectors.ts | 9 ++-- .../redux-slices/selectors/uiSelectors.ts | 5 +- 6 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 background/redux-slices/networks.ts diff --git a/background/main.ts b/background/main.ts index c3ff3844a5..c498e628f3 100644 --- a/background/main.ts +++ b/background/main.ts @@ -29,7 +29,6 @@ import { AddressOnNetwork, NameOnNetwork } from "./accounts" import rootReducer from "./redux-slices" import { loadAccount, - blockSeen, updateAccountBalance, updateENSName, updateENSAvatar, @@ -43,6 +42,7 @@ import { updateKeyrings, setKeyringToVerify, } from "./redux-slices/keyrings" +import { blockSeen } from "./redux-slices/networks" import { initializationLoadingTimeHitLimit, emitter as uiSliceEmitter, @@ -101,7 +101,7 @@ const devToolsSanitizer = (input: unknown) => { // The version of persisted Redux state the extension is expecting. Any previous // state without this version, or with a lower version, ought to be migrated. -const REDUX_STATE_VERSION = 3 +const REDUX_STATE_VERSION = 4 type Migration = (prevState: Record) => Record @@ -141,6 +141,54 @@ const REDUX_MIGRATIONS: { [version: number]: Migration } = { return newState }, + 4: (prevState: Record) => { + // Migrate the ETH-only block data in store.accounts.blocks[blockHeight] to + // a new networks slice. Block data is now network-specific, keyed by EVM + // chainID in store.networks.networkData[chainId].blocks + type OldState = { + account?: { + blocks?: { [blockHeight: number]: unknown } + } + } + type NetworkState = { + evm: { + [chainID: string]: { + blockHeight: number | null + blocks: { + [blockHeight: number]: unknown + } + } + } + } + + const oldState = prevState as OldState + + const networks: NetworkState = { + evm: { + "1": { + blocks: { ...oldState.account?.blocks }, + blockHeight: + Math.max( + ...Object.keys(oldState.account?.blocks ?? {}).map((s) => + parseInt(s, 10) + ) + ) || null, + }, + }, + } + + const { blocks, ...oldStateAccountWithoutBlocks } = oldState.account ?? { + blocks: undefined, + } + + return { + ...prevState, + // Drop blocks from account slice. + account: oldStateAccountWithoutBlocks, + // Add new networks slice data. + networks, + } + }, } // Migrate a previous version of the Redux state to that expected by the current diff --git a/background/redux-slices/accounts.ts b/background/redux-slices/accounts.ts index 3a906624e1..5f0cfa9797 100644 --- a/background/redux-slices/accounts.ts +++ b/background/redux-slices/accounts.ts @@ -1,7 +1,7 @@ import { createSlice } from "@reduxjs/toolkit" import { createBackgroundAsyncThunk } from "./utils" import { AccountBalance, AddressOnNetwork, NameOnNetwork } from "../accounts" -import { AnyEVMBlock, Network } from "../networks" +import { Network } from "../networks" import { AnyAsset, AnyAssetAmount, SmartContractFungibleAsset } from "../assets" import { AssetMainCurrencyAmount, @@ -51,9 +51,6 @@ export type AccountState = { // TODO Adapt to use AccountNetwork, probably via a Map and custom serialization/deserialization. accountsData: { [address: string]: AccountData | "loading" } combinedData: CombinedAccountData - // TODO the blockHeight key should be changed to something - // compatible with the idea of multiple networks. - blocks: { [blockHeight: number]: AnyEVMBlock } } export type CombinedAccountData = { @@ -83,7 +80,6 @@ export const initialState = { totalMainCurrencyValue: "", assets: [], }, - blocks: {}, } as AccountState function newAccountData( @@ -140,9 +136,6 @@ const accountSlice = createSlice({ name: "account", initialState, reducers: { - blockSeen: (immerState, { payload: block }: { payload: AnyEVMBlock }) => { - immerState.blocks[block.blockHeight] = block - }, loadAccount: (state, { payload: accountToLoad }: { payload: string }) => { return state.accountsData[accountToLoad] ? state // If the account data already exists, the account is already loaded. @@ -256,7 +249,6 @@ export const { updateAccountBalance, updateENSName, updateENSAvatar, - blockSeen, } = accountSlice.actions export default accountSlice.reducer diff --git a/background/redux-slices/index.ts b/background/redux-slices/index.ts index c96941ddc1..d2699acf1f 100644 --- a/background/redux-slices/index.ts +++ b/background/redux-slices/index.ts @@ -6,6 +6,7 @@ import accountsReducer from "./accounts" import assetsReducer from "./assets" import activitiesReducer from "./activities" import keyringsReducer from "./keyrings" +import networksReducer from "./networks" import swapReducer from "./0x-swap" import transactionConstructionReducer from "./transaction-construction" import uiReducer from "./ui" @@ -19,6 +20,7 @@ const mainReducer = combineReducers({ assets: assetsReducer, activities: activitiesReducer, keyrings: keyringsReducer, + networks: networksReducer, swap: swapReducer, transactionConstruction: transactionConstructionReducer, ui: uiReducer, diff --git a/background/redux-slices/networks.ts b/background/redux-slices/networks.ts new file mode 100644 index 0000000000..b7700ee9b7 --- /dev/null +++ b/background/redux-slices/networks.ts @@ -0,0 +1,48 @@ +import { createSlice } from "@reduxjs/toolkit" + +import { AnyEVMBlock } from "../networks" + +type NetworkState = { + blocks: { [blockHeight: number]: AnyEVMBlock } + blockHeight: number | null +} + +export type NetworksState = { + evm: { + [chainID: string]: NetworkState + } +} + +export const initialState: NetworksState = { + evm: { + "1": { + blockHeight: null, + blocks: {}, + }, + }, +} + +const networksSlice = createSlice({ + name: "networks", + initialState, + reducers: { + blockSeen: (immerState, { payload: block }: { payload: AnyEVMBlock }) => { + if (!(block.network.chainID in immerState.evm)) { + immerState.evm[block.network.chainID] = { + blocks: {}, + blockHeight: block.blockHeight, + } + } else if ( + block.blockHeight > + (immerState.evm[block.network.chainID].blockHeight || 0) + ) { + immerState.evm[block.network.chainID].blockHeight = block.blockHeight + } + immerState.evm[block.network.chainID].blocks[block.blockHeight] = block + }, + }, +}) + +export const { blockSeen } = networksSlice.actions + +export default networksSlice.reducer diff --git a/background/redux-slices/selectors/activitiesSelectors.ts b/background/redux-slices/selectors/activitiesSelectors.ts index a8e4977b53..9c3bce2c42 100644 --- a/background/redux-slices/selectors/activitiesSelectors.ts +++ b/background/redux-slices/selectors/activitiesSelectors.ts @@ -5,14 +5,13 @@ import { RootState } from ".." export const selectCurrentAccountActivitiesWithTimestamps = createSelector( (state: RootState) => { - const currentAccountAddress = selectCurrentAccount(state).address + const currentAccount = selectCurrentAccount(state) + const { address, network } = currentAccount return { currentAccountActivities: - typeof currentAccountAddress !== "undefined" - ? state.activities[currentAccountAddress] - : undefined, - blocks: state.account.blocks, + typeof address !== "undefined" ? state.activities[address] : undefined, + blocks: state.networks.evm[network.chainID]?.blocks ?? {}, } }, ({ currentAccountActivities, blocks }) => { diff --git a/background/redux-slices/selectors/uiSelectors.ts b/background/redux-slices/selectors/uiSelectors.ts index d0e4f1bfe4..6f1b6faa50 100644 --- a/background/redux-slices/selectors/uiSelectors.ts +++ b/background/redux-slices/selectors/uiSelectors.ts @@ -8,7 +8,10 @@ const hardcodedMainCurrencySymbol = "USD" export const selectShowingActivityDetail = createSelector( (state: RootState) => state.activities, (state: RootState) => state.ui.showingActivityDetailID, - (state: RootState) => state.account.blocks, + (state: RootState) => { + const { network } = state.ui.selectedAccount + return state.networks.evm[network.chainID].blocks + }, (activities, showingActivityDetailID, blocks) => { return showingActivityDetailID === null ? null From 056a81506571d42f0d2dab5081671fa7f9d152d6 Mon Sep 17 00:00:00 2001 From: Antonio Salazar Cardozo Date: Tue, 22 Feb 2022 14:29:39 -0500 Subject: [PATCH 2/4] Explicitly resolve network for transactions Currently the ChainService only returns the one Ethereum network it's connected to, and the callers throw if the service is not connected to the chain id specified in the transaction; however, the intent is to implement `resolveNetwork` further to recognize multiple networks when available. --- background/main.ts | 6 +++++- background/services/chain/index.ts | 9 +++++++++ background/services/signing/index.ts | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/background/main.ts b/background/main.ts index c498e628f3..675342167e 100644 --- a/background/main.ts +++ b/background/main.ts @@ -664,6 +664,11 @@ export default class Main extends BaseService { "requestSignature", async ({ transaction, method }) => { if (HIDE_IMPORT_LEDGER) { + const network = this.chainService.resolveNetwork(transaction) + if (typeof network === "undefined") { + throw new Error(`Unknown chain ID ${transaction.chainID}.`) + } + const transactionWithNonce = await this.chainService.populateEVMTransactionNonce(transaction) @@ -686,7 +691,6 @@ export default class Main extends BaseService { } else { try { const signedTx = await this.signingService.signTransaction( - this.chainService.ethereumNetwork, transaction, method ) diff --git a/background/services/chain/index.ts b/background/services/chain/index.ts index fadae70bea..3d696be4eb 100644 --- a/background/services/chain/index.ts +++ b/background/services/chain/index.ts @@ -391,6 +391,15 @@ export default class ChainService extends BaseService { } } + resolveNetwork( + transactionRequest: EIP1559TransactionRequest + ): EVMNetwork | undefined { + if (transactionRequest.chainID === this.ethereumNetwork.chainID) { + return this.ethereumNetwork + } + return undefined + } + /** * Releases the specified nonce for the given network and address. This * updates internal service state to allow that nonce to be reused. In cases diff --git a/background/services/signing/index.ts b/background/services/signing/index.ts index 3ba1f3aa2a..6f8f134f15 100644 --- a/background/services/signing/index.ts +++ b/background/services/signing/index.ts @@ -111,10 +111,14 @@ export default class SigningService extends BaseService { } async signTransaction( - network: EVMNetwork, transactionRequest: EIP1559TransactionRequest, signingMethod: SigningMethod ): Promise { + const network = this.chainService.resolveNetwork(transactionRequest) + if (typeof network === "undefined") { + throw new Error(`Unknown chain ID ${transactionRequest.chainID}.`) + } + const transactionWithNonce = await this.chainService.populateEVMTransactionNonce(transactionRequest) From 76ef8b8e0e9c1f2eaf2c253c1b44d21a40c67165 Mon Sep 17 00:00:00 2001 From: Antonio Salazar Cardozo Date: Tue, 22 Feb 2022 14:30:52 -0500 Subject: [PATCH 3/4] SigningService no longer releases successful nonces The SigningService was set up to release nonces in a finally block during signing, but a successful signature should continue to reserve the nonce. Only a failed signature should release it. --- background/services/signing/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/background/services/signing/index.ts b/background/services/signing/index.ts index 6f8f134f15..5ab92bf290 100644 --- a/background/services/signing/index.ts +++ b/background/services/signing/index.ts @@ -155,9 +155,9 @@ export default class SigningService extends BaseService { reason: "genericError", }) - throw err - } finally { this.chainService.releaseEVMTransactionNonce(transactionWithNonce) + + throw err } } From 561fd251a6de27e473fbd22d22a9e8a6ebbcd1b2 Mon Sep 17 00:00:00 2001 From: Antonio Salazar Cardozo Date: Thu, 24 Feb 2022 15:56:10 -0500 Subject: [PATCH 4/4] Add SerialFallbackProvider for Ethers SerialFallbackProvider is an Ethers JsonRpcProvider that is built to fall back among a list of providers that could fulfill requests. By default, it always uses a WebSocketProvider, but can fall back to other providers that may or may not use WebSockets if that provider fails. Fallbacks only kick in after exponentially backed-off retries, and retries can, for WebSockets, reconnect the WebSocket. Additionally, the fallback provider tracks subscriptions and reattaches them when a provider switches or is reconnected. The serial fallback provider will also be the place where abstractions will occur between different types of providers, e.g. using optimized Alchemy-specific calls for an Alchemy provider but less optimized ones for non-Alchemy providers. --- background/main.ts | 2 +- background/services/chain/index.ts | 100 ++-- .../chain/serial-fallback-provider.ts | 466 ++++++++++++++++++ background/services/indexing/index.ts | 8 +- background/services/name/index.ts | 6 +- 5 files changed, 524 insertions(+), 58 deletions(-) create mode 100644 background/services/chain/serial-fallback-provider.ts diff --git a/background/main.ts b/background/main.ts index 675342167e..d8973c8d13 100644 --- a/background/main.ts +++ b/background/main.ts @@ -597,7 +597,7 @@ export default class Main extends BaseService { async getAccountEthBalanceUncached(address: string): Promise { const amountBigNumber = - await this.chainService.pollingProviders.ethereum.getBalance(address) + await this.chainService.providers.ethereum.getBalance(address) return amountBigNumber.toBigInt() } diff --git a/background/services/chain/index.ts b/background/services/chain/index.ts index 3d696be4eb..709b12f4f5 100644 --- a/background/services/chain/index.ts +++ b/background/services/chain/index.ts @@ -1,6 +1,7 @@ import { AlchemyProvider, AlchemyWebSocketProvider, + BaseProvider, TransactionReceipt, } from "@ethersproject/providers" import { getNetwork } from "@ethersproject/networks" @@ -48,6 +49,7 @@ import type { EnrichedEVMTransactionSignatureRequest, } from "../enrichment" import { HOUR } from "../../constants" +import SerialFallbackProvider from "./serial-fallback-provider" // We can't use destructuring because webpack has to replace all instances of // `process.env` variables in the bundled output @@ -114,18 +116,16 @@ interface Events extends ServiceLifecycleEvents { * case a service needs to interact with a network directly. */ export default class ChainService extends BaseService { - pollingProviders: { [networkName: string]: AlchemyProvider } - - websocketProviders: { [networkName: string]: AlchemyWebSocketProvider } + providers: { [networkName: string]: SerialFallbackProvider } subscribedAccounts: { account: string - provider: AlchemyWebSocketProvider + provider: SerialFallbackProvider }[] subscribedNetworks: { network: EVMNetwork - provider: AlchemyWebSocketProvider + provider: SerialFallbackProvider }[] /** @@ -195,18 +195,21 @@ export default class ChainService extends BaseService { this.ethereumNetwork = getEthereumNetwork() // TODO set up for each relevant network - this.pollingProviders = { - ethereum: new AlchemyProvider( - getNetwork(Number(this.ethereumNetwork.chainID)), - ALCHEMY_KEY - ), - } - this.websocketProviders = { - ethereum: new AlchemyWebSocketProvider( - getNetwork(Number(this.ethereumNetwork.chainID)), - ALCHEMY_KEY + this.providers = { + ethereum: new SerialFallbackProvider( + () => + new AlchemyWebSocketProvider( + getNetwork(Number(this.ethereumNetwork.chainID)), + ALCHEMY_KEY + ), + () => + new AlchemyProvider( + getNetwork(Number(this.ethereumNetwork.chainID)), + ALCHEMY_KEY + ) ), } + this.subscribedAccounts = [] this.subscribedNetworks = [] this.transactionsToRetrieve = { ethereum: [] } @@ -216,7 +219,7 @@ export default class ChainService extends BaseService { await super.internalStartService() const accounts = await this.getAccountsToTrack() - const ethProvider = this.pollingProviders.ethereum + const ethProvider = this.providers.ethereum const network = this.ethereumNetwork // FIXME Should we await or drop Promise.all on the below two? @@ -350,7 +353,7 @@ export default class ChainService extends BaseService { const normalizedAddress = normalizeEVMAddress(transactionRequest.from) const chainNonce = - (await this.pollingProviders.ethereum.getTransactionCount( + (await this.providers.ethereum.getTransactionCount( transactionRequest.from, "latest" )) - 1 @@ -455,7 +458,7 @@ export default class ChainService extends BaseService { this.checkNetwork(addressNetwork.network) // TODO look up provider network properly - const balance = await this.pollingProviders.ethereum.getBalance( + const balance = await this.providers.ethereum.getBalance( addressNetwork.address ) const accountBalance: AccountBalance = { @@ -487,7 +490,7 @@ export default class ChainService extends BaseService { return cachedBlock.blockHeight } // TODO make proper use of the network - return this.pollingProviders.ethereum.getBlockNumber() + return this.providers.ethereum.getBlockNumber() } /** @@ -510,7 +513,7 @@ export default class ChainService extends BaseService { } // Looking for new block - const resultBlock = await this.pollingProviders.ethereum.getBlock(blockHash) + const resultBlock = await this.providers.ethereum.getBlock(blockHash) const block = blockFromEthersBlock(network, resultBlock) @@ -538,9 +541,7 @@ export default class ChainService extends BaseService { return cachedTx } // TODO make proper use of the network - const gethResult = await this.pollingProviders.ethereum.getTransaction( - txHash - ) + const gethResult = await this.providers.ethereum.getTransaction(txHash) const newTransaction = transactionFromEthersTransaction( gethResult, ETH, @@ -591,7 +592,7 @@ export default class ChainService extends BaseService { network: EVMNetwork, transactionRequest: EIP1559TransactionRequest ): Promise { - const estimate = await this.pollingProviders.ethereum.estimateGas( + const estimate = await this.providers.ethereum.estimateGas( ethersTransactionRequestFromEIP1559TransactionRequest(transactionRequest) ) // Add 10% more gas as a safety net @@ -615,23 +616,21 @@ export default class ChainService extends BaseService { ) try { await Promise.all([ - this.pollingProviders.ethereum - .sendTransaction(serialized) - .catch((error) => { - logger.debug( - "Broadcast error caught, saving failed status and releasing nonce...", - transaction, - error - ) - // Failure to broadcast needs to be registered. - this.saveTransaction( - { ...transaction, status: 0, error: error.toString() }, - "alchemy" - ) - this.releaseEVMTransactionNonce(transaction) - - return Promise.reject(error) - }), + this.providers.ethereum.sendTransaction(serialized).catch((error) => { + logger.debug( + "Broadcast error caught, saving failed status and releasing nonce...", + transaction, + error + ) + // Failure to broadcast needs to be registered. + this.saveTransaction( + { ...transaction, status: 0, error: error.toString() }, + "alchemy" + ) + this.releaseEVMTransactionNonce(transaction) + + return Promise.reject(error) + }), this.subscribeToTransactionConfirmation( transaction.network, transaction @@ -659,7 +658,7 @@ export default class ChainService extends BaseService { } async send(method: string, params: unknown[]): Promise { - return this.websocketProviders.ethereum.send(method, params) + return this.providers.ethereum.send(method, params) } /* ***************** @@ -789,7 +788,7 @@ export default class ChainService extends BaseService { // TODO only works on Ethereum today const assetTransfers = await getAssetTransfers( - this.pollingProviders.ethereum, + this.providers.ethereum as unknown as AlchemyWebSocketProvider, addressOnNetwork, Number(startBlock), Number(endBlock) @@ -841,7 +840,7 @@ export default class ChainService extends BaseService { toHandle.forEach(async ({ hash, firstSeen }) => { try { // TODO make this multi network - const result = await this.pollingProviders.ethereum.getTransaction(hash) + const result = await this.providers.ethereum.getTransaction(hash) const transaction = transactionFromEthersTransaction( result, @@ -983,9 +982,9 @@ export default class ChainService extends BaseService { */ private async subscribeToNewHeads(network: EVMNetwork): Promise { // TODO look up provider network properly - const provider = this.websocketProviders.ethereum + const provider = this.providers.ethereum // eslint-disable-next-line no-underscore-dangle - await provider._subscribe( + await provider.subscribe( "newHeadsSubscriptionID", ["newHeads"], async (result: unknown) => { @@ -1018,9 +1017,8 @@ export default class ChainService extends BaseService { this.checkNetwork(network) // TODO look up provider network properly - const provider = this.websocketProviders.ethereum - // eslint-disable-next-line no-underscore-dangle - await provider._subscribe( + const provider = this.providers.ethereum + await provider.subscribe( "filteredNewFullPendingTransactionsSubscriptionID", ["alchemy_filteredNewFullPendingTransactions", { address }], async (result: unknown) => { @@ -1082,7 +1080,7 @@ export default class ChainService extends BaseService { this.checkNetwork(network) // TODO make proper use of the network - this.websocketProviders.ethereum.once( + this.providers.ethereum.once( transaction.hash, (confirmedReceipt: TransactionReceipt) => { this.saveTransaction( @@ -1106,7 +1104,7 @@ export default class ChainService extends BaseService { this.checkNetwork(network) // TODO make proper use of the network - const receipt = await this.pollingProviders.ethereum.getTransactionReceipt( + const receipt = await this.providers.ethereum.getTransactionReceipt( transaction.hash ) await this.saveTransaction( diff --git a/background/services/chain/serial-fallback-provider.ts b/background/services/chain/serial-fallback-provider.ts new file mode 100644 index 0000000000..12f9d673f4 --- /dev/null +++ b/background/services/chain/serial-fallback-provider.ts @@ -0,0 +1,466 @@ +import { Network } from "@ethersproject/networks" +import { + EventType, + JsonRpcProvider, + Listener, + WebSocketProvider, +} from "@ethersproject/providers" +import { MINUTE } from "../../constants" +import logger from "../../lib/logger" + +// Back off by this amount as a base, exponentiated by attempts and jittered. +const BASE_BACKOFF_MS = 150 +// Reset backoffs after 5 minutes. +const COOLDOWN_PERIOD = 5 * MINUTE +// Retry 3 times before falling back to the next provider. +const MAX_RETRIES = 3 + +/** + * Wait the given number of ms, then run the provided function. Returns a + * promise that will resolve after the delay has elapsed and the passed + * function has executed, with the result of the passed function. + */ +function waitAnd>( + waitMs: number, + fn: () => E +): Promise { + return new Promise((resolve) => { + // TODO setTimeout rather than browser.alarms here could mean this would + // hang when transitioning to a transient background page? Can we do this + // with browser.alarms? + setTimeout(() => { + resolve(fn()) + }, waitMs) + }) +} + +/** + * Given the number of the backoff being executed, returns a jittered number of + * ms to back off before making the next attempt. + */ +function backedOffMs(backoffCount: number): number { + const backoffSlotStart = BASE_BACKOFF_MS ** backoffCount + const backoffSlotEnd = BASE_BACKOFF_MS ** (backoffCount + 1) + + return backoffSlotStart + Math.random() * (backoffSlotEnd - backoffSlotStart) +} + +/** + * Returns true if the given provider is using a WebSocket AND the WebSocket is + * either closing or already closed. Ethers does not provide direct access to + * this information, nor does it attempt to reconnect in these cases. + */ +function isClosedOrClosingWebSocketProvider( + provider: JsonRpcProvider +): boolean { + if (provider instanceof WebSocketProvider) { + // Digging into the innards of Ethers here because there's no + // other way to get access to the WebSocket connection situation. + // eslint-disable-next-line no-underscore-dangle + const webSocket = provider._websocket as WebSocket + + return ( + webSocket.readyState === WebSocket.CLOSING || + webSocket.readyState === WebSocket.CLOSED + ) + } + + return false +} + +/** + * The SerialFallbackProvider is an Ethers JsonRpcProvider that can fall back + * through a series of providers in case previous ones fail. + * + * In case of server errors, this provider attempts a number of exponential + * backoffs and retries before falling back to the next provider in the list. + * WebSocketProviders in the list are checked for WebSocket connections, and + * attempt reconnects if the underlying WebSocket disconnects. + * + * Additionally, subscriptions are tracked and, if the current provider is a + * WebSocket provider, they are restored on reconnect. + */ +export default class SerialFallbackProvider extends JsonRpcProvider { + // Functions that will create and initialize a new provider, in priority + // order. + private providerCreators: [ + () => WebSocketProvider, + ...(() => JsonRpcProvider)[] + ] + + // The currently-used provider, produced by the provider-creator at + // currentProviderIndex. + private currentProvider: JsonRpcProvider + + // The index of the provider creator that created the current provider. Used + // for reconnects when relevant. + private currentProviderIndex = 0 + + // Information on the current backoff state. This is used to ensure retries + // and reconnects back off exponentially. + private currentBackoff = { + providerIndex: 0, + backoffMs: BASE_BACKOFF_MS, + backoffCount: 0, + lastBackoffTime: 0, + } + + // Information on WebSocket-style subscriptions. Tracked here so as to + // restore them in case of WebSocket disconnects. + private subscriptions: { + tag: string + param: unknown[] + processFunc: (result: unknown) => void + }[] = [] + + // Information on event subscriptions, which can be restored on non-WebSocket + // subscriptions and WebSocket subscriptions both. + private eventSubscriptions: { + eventName: EventType + listener: Listener | (Listener & { wrappedListener: Listener }) + once: boolean + }[] = [] + + constructor( + firstProviderCreator: () => WebSocketProvider, + ...remainingProviderCreators: (() => JsonRpcProvider)[] + ) { + const firstProvider = firstProviderCreator() + + super(firstProvider.connection, firstProvider.network) + + this.currentProvider = firstProvider + this.providerCreators = [firstProviderCreator, ...remainingProviderCreators] + } + + /** + * Override the core `perform` method to handle disconnects and other errors + * that should trigger retries. Ethers already does internal retrying, but + * this retry methodology eventually falls back on another provider, handles + * WebSocket disconnects, and restores subscriptions where + * possible/necessary. + */ + async send(method: string, params: unknown): Promise { + try { + if (isClosedOrClosingWebSocketProvider(this.currentProvider)) { + // Detect disconnected WebSocket and immediately throw. + throw new Error("WebSocket is already in CLOSING") + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return await this.currentProvider.send(method, params as any) + } catch (error) { + // Awful, but what can ya do. + const stringifiedError = String(error) + + if ( + stringifiedError.match(/WebSocket is already in CLOSING|bad response/) + ) { + const backoff = this.backoffFor(this.currentProviderIndex) + if (typeof backoff === "undefined") { + logger.debug( + "Attempting to connect new provider after error", + error, + "." + ) + this.disconnectCurrentProvider() + + this.currentProviderIndex += 1 + if (this.currentProviderIndex < this.providerCreators.length) { + // Try again with the next provider. + await this.reconnectProvider() + + return await this.send(method, params) + } + + // If we've looped around, set us up for the next call, but fail the + // current one since we've gone through every available provider. Note + // that this may happen over time, but we still fail the request that + // hits the end of the list. + this.currentProviderIndex = 0 + + // Reconnect, but don't wait for the connection to go through. + this.reconnectProvider() + + throw error + } else { + logger.debug( + "Backing off for", + backoff, + "and retrying: ", + method, + params + ) + + return await waitAnd(backoff, async () => { + if (isClosedOrClosingWebSocketProvider(this.currentProvider)) { + await this.reconnectProvider() + } + + return this.send(method, params) + }) + } + } + + logger.debug( + "Skipping fallback for unidentified error", + error, + "for provider", + this.currentProvider + ) + + throw error + } + } + + private async disconnectCurrentProvider() { + logger.debug( + "Disconnecting current provider; websocket: ", + this.currentProvider instanceof WebSocketProvider, + "." + ) + if (this.currentProvider instanceof WebSocketProvider) { + this.currentProvider.destroy() + } else { + // For non-WebSocket providers, kill all subscriptions so the listeners + // won't fire; the next provider will pick them up. We could lose events + // in between, but if we're considering the current provider dead, let's + // assume we would lose them anyway. + this.eventSubscriptions.forEach(({ eventName }) => + this.removeAllListeners(eventName) + ) + } + } + + async subscribe( + tag: string, + param: Array, + processFunc: (result: unknown) => void + ): Promise { + const subscription = { tag, param, processFunc } + this.subscriptions.push(subscription) + + if (this.currentProvider instanceof WebSocketProvider) { + // eslint-disable-next-line no-underscore-dangle + await this.currentProvider._subscribe(tag, param, processFunc) + } else { + logger.warn( + "Current provider is not a WebSocket provider; subscription " + + "will not work until a WebSocket provider connects." + ) + } + } + + async detectNetwork(): Promise { + return this.currentProvider.detectNetwork() + } + + // Overriding internal functionality here to support event listener + // restoration on reconnect. + // eslint-disable-next-line no-underscore-dangle + on(eventName: EventType, listener: Listener): this { + this.eventSubscriptions.push({ + eventName, + listener, + once: false, + }) + + this.currentProvider.on(eventName, listener) + + return this + } + + once(eventName: EventType, listener: Listener): this { + const adjustedListener = this.listenerWithCleanup(eventName, listener) + + this.eventSubscriptions.push({ + eventName, + listener: adjustedListener, + once: true, + }) + + this.currentProvider.once(eventName, listener) + + return this + } + + /** + * Removes one or all listeners for a given event. + * + * Ensures these will not be restored during a reconnect. + */ + off(eventName: EventType, listenerToRemove?: Listener): this { + this.eventSubscriptions = this.eventSubscriptions.filter( + ({ eventName: savedEventName, listener: savedListener }) => { + if (savedEventName === eventName) { + // No explicit listener to remove = remove all listeners. + if ( + typeof listenerToRemove === "undefined" || + listenerToRemove === null + ) { + return true + } + + // If the listener is wrapped, use that to check against the + // specified listener to remove. + if ("wrappedListener" in savedListener) { + return savedListener.wrappedListener === listenerToRemove + } + + // Otherwise, directly compare. + return savedListener === listenerToRemove + } + + return false + } + ) + + this.currentProvider.off(eventName, listenerToRemove) + + return this + } + + /** + * Wraps an Ethers listener function meant to only be invoked once with + * cleanup to ensure it won't be resubscribed in case of a provider switch. + */ + private listenerWithCleanup( + eventName: EventType, + listenerToWrap: Listener + ): Listener & { wrappedListener: Listener } { + const wrappedListener = ( + ...params: Parameters + ): ReturnType => { + try { + listenerToWrap(...params) + } finally { + this.eventSubscriptions = this.eventSubscriptions.filter( + ({ eventName: storedEventName, listener, once }) => + eventName !== storedEventName || + listener !== wrappedListener || + once !== true + ) + } + } + + wrappedListener.wrappedListener = listenerToWrap + + return wrappedListener + } + + /** + * Reconnects the currently-selected provider. If the current provider index + * has been somehow set out of range, resets it to 0. + */ + private async reconnectProvider() { + if (this.currentProviderIndex >= this.providerCreators.length) { + this.currentProviderIndex = 0 + } + + logger.debug( + "Reconnecting provider at index", + this.currentProviderIndex, + "..." + ) + + this.currentProvider = this.providerCreators[this.currentProviderIndex]() + this.resubscribe() + + // TODO After a longer backoff, attempt to reset the current provider to 0. + } + + private async resubscribe() { + logger.debug("Resubscribing subscriptions...") + + if (this.currentProvider instanceof WebSocketProvider) { + const provider = this.currentProvider as WebSocketProvider + + // Chain promises to serially resubscribe. + // + // TODO If anything fails along the way, it should yield the same kind of + // TODO backoff as a regular `perform`. + await this.subscriptions.reduce( + (previousPromise, { tag, param, processFunc }) => + previousPromise.then(() => + waitAnd(backedOffMs(0), () => + // Direct subscriptions are internal, but we want to be able to + // restore them. + // eslint-disable-next-line no-underscore-dangle + provider._subscribe(tag, param, processFunc) + ) + ), + Promise.resolve() + ) + } else if (this.subscriptions.length > 0) { + logger.warn( + `Cannot resubscribe ${this.subscriptions.length} subscription(s) ` + + `as the current provider is not a WebSocket provider; waiting ` + + `until a WebSocket provider connects to restore subscriptions ` + + `properly.` + ) + } + + this.eventSubscriptions.forEach(({ eventName, listener, once }) => { + if (once) { + this.currentProvider.once(eventName, listener) + } else { + this.currentProvider.on(eventName, listener) + } + }) + + logger.debug("Subscriptions resubscribed...") + } + + /** + * Computes the backoff time for the given provider index. If the provider + * index is new, starts with the base backoff; if the provider index is + * unchanged, computes a jittered exponential backoff. If the current + * provider has already exceeded its maximum retries, returns undefined to + * signal the provider should be considered dead for the time being. + * + * Backoffs respect a cooldown time after which they reset down to the base + * backoff time. + */ + private backoffFor(providerIndex: number): number | undefined { + const { + providerIndex: existingProviderIndex, + backoffCount, + lastBackoffTime, + } = this.currentBackoff + + if (backoffCount > MAX_RETRIES) { + return undefined + } + + if (existingProviderIndex !== providerIndex) { + this.currentBackoff = { + providerIndex, + backoffMs: BASE_BACKOFF_MS, + backoffCount: 0, + lastBackoffTime: 0, + } + } else if (Date.now() - lastBackoffTime > COOLDOWN_PERIOD) { + this.currentBackoff = { + providerIndex, + backoffMs: BASE_BACKOFF_MS, + backoffCount: 0, + lastBackoffTime: 0, + } + } else { + // The next backoff slot starts at the current minimum backoff and + // extends until the start of the next backoff. This specific backoff is + // randomized within that slot. + const newBackoffCount = backoffCount + 1 + const backoffMs = backedOffMs(newBackoffCount) + + this.currentBackoff = { + providerIndex, + backoffMs, + backoffCount: newBackoffCount, + lastBackoffTime: Date.now(), + } + } + + return this.currentBackoff.backoffMs + } +} diff --git a/background/services/indexing/index.ts b/background/services/indexing/index.ts index 2165c3561f..98ff6e0543 100644 --- a/background/services/indexing/index.ts +++ b/background/services/indexing/index.ts @@ -1,3 +1,4 @@ +import { AlchemyProvider } from "@ethersproject/providers" import logger from "../../lib/logger" import { HexString } from "../../types" @@ -337,7 +338,7 @@ export default class IndexingService extends BaseService { contractAddresses?: HexString[] ): ReturnType { const balances = await getTokenBalances( - this.chainService.pollingProviders.ethereum, + this.chainService.providers.ethereum as unknown as AlchemyProvider, addressNetwork.address, contractAddresses || undefined ) @@ -414,7 +415,8 @@ export default class IndexingService extends BaseService { ) if (!customAsset) { // TODO hardcoded to Ethereum - const provider = this.chainService.pollingProviders.ethereum + const provider = this.chainService.providers + .ethereum as unknown as AlchemyProvider // pull metadata from Alchemy customAsset = (await getTokenMetadata(provider, { @@ -586,7 +588,7 @@ export default class IndexingService extends BaseService { ).map(async (addressOnNetwork) => { // TODO hardcoded to Ethereum const balances = await getAssetBalances( - this.chainService.pollingProviders.ethereum, + this.chainService.providers.ethereum as unknown as AlchemyProvider, activeAssetsToTrack, addressOnNetwork ) diff --git a/background/services/name/index.ts b/background/services/name/index.ts index 9c53a88fe4..fc69f3ed64 100644 --- a/background/services/name/index.ts +++ b/background/services/name/index.ts @@ -163,7 +163,7 @@ export default class NameService extends BaseService { // TODO ENS lookups should work on Ethereum mainnet and a few testnets as well. // This is going to be strange, though, as we'll be looking up ENS names for // non-Ethereum networks (eg eventually Bitcoin). - const provider = this.chainService.pollingProviders.ethereum + const provider = this.chainService.providers.ethereum // TODO cache name resolution and TTL const address = await provider.resolveName(name) if (!address || !address.match(/^0x[a-zA-Z0-9]*$/)) { @@ -198,7 +198,7 @@ export default class NameService extends BaseService { } } - const provider = this.chainService.pollingProviders.ethereum + const provider = this.chainService.providers.ethereum // TODO cache name resolution and TTL const name = await provider.lookupAddress(address) // TODO proper domain name validation ala RFC2181 @@ -248,7 +248,7 @@ export default class NameService extends BaseService { return undefined } // TODO handle if it doesn't exist - const provider = this.chainService.pollingProviders.ethereum + const provider = this.chainService.providers.ethereum const resolver = await provider.getResolver(name) if (!sameEVMAddress(await resolver?.getAddress(), address)) { return undefined