From f3e6192d6910dba39941d7817c818234edeeceff Mon Sep 17 00:00:00 2001 From: Samuel Holmes Date: Thu, 7 Sep 2023 15:54:48 -0700 Subject: [PATCH 1/6] Fix typo in asFilfoxMessage cleaner --- src/filecoin/Filfox.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filecoin/Filfox.ts b/src/filecoin/Filfox.ts index 4f4d79c7c..db9bc3e6b 100644 --- a/src/filecoin/Filfox.ts +++ b/src/filecoin/Filfox.ts @@ -40,7 +40,7 @@ export const asFilfoxMessage = asObject({ height: asNumber, method: asString, nonce: asNumber, - reciept: asObject({ + receipt: asObject({ exitCode: asNumber }), timestamp: asNumber, From 0fcb0d25cab0f3c76186479cc7bb808e5e4915b9 Mon Sep 17 00:00:00 2001 From: Samuel Holmes Date: Fri, 8 Sep 2023 15:02:45 -0700 Subject: [PATCH 2/6] Add `isScanning` flag to prevent double-scanning during long syncs --- src/filecoin/FilecoinEngine.ts | 98 +++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 43 deletions(-) diff --git a/src/filecoin/FilecoinEngine.ts b/src/filecoin/FilecoinEngine.ts index 636ddccc9..5da43bbb5 100644 --- a/src/filecoin/FilecoinEngine.ts +++ b/src/filecoin/FilecoinEngine.ts @@ -48,6 +48,7 @@ export class FilecoinEngine extends CurrencyEngine< > { address: Address availableAttoFil: string + isScanning: boolean networkInfo: FilecoinNetworkInfo otherData!: FilecoinWalletOtherData pluginId: string @@ -68,6 +69,10 @@ export class FilecoinEngine extends CurrencyEngine< const { networkInfo } = env this.address = Address.fromString(walletInfo.keys.address) this.availableAttoFil = '0' + this.isScanning = false + this.networkInfo = networkInfo + this.pluginId = this.currencyInfo.pluginId + this.filRpc = new RPC(env.networkInfo.rpcNode.networkName, { url: env.networkInfo.rpcNode.url, token: env.currencyInfo.currencyCode @@ -75,9 +80,6 @@ export class FilecoinEngine extends CurrencyEngine< this.filfoxApi = new Filfox(env.networkInfo.filfoxUrl, env.io.fetchCors) this.filscanApi = new Filscan(env.networkInfo.filscanUrl, env.io.fetchCors) this.rpcExtra = new RpcExtra(env.networkInfo.rpcNode.url, env.io.fetchCors) - - this.networkInfo = networkInfo - this.pluginId = this.currencyInfo.pluginId } setOtherData(raw: any): void { @@ -299,52 +301,62 @@ export class FilecoinEngine extends CurrencyEngine< } async checkTransactions(): Promise { - const addressString = this.address.toString() - - const handleScanProgress = (progress: number): void => { - const currentProgress = - this.tokenCheckTransactionsStatus[this.currencyInfo.currencyCode] - const newProgress = progress - - if ( - // Only send event if we haven't completed sync - currentProgress < 1 && - // Avoid thrashing - (newProgress >= 1 || newProgress > currentProgress * 1.1) - ) { - this.tokenCheckTransactionsStatus[this.currencyInfo.currencyCode] = - newProgress - this.updateOnAddressesChecked() + if (this.isScanning) return + try { + this.isScanning = true + + const addressString = this.address.toString() + + const handleScanProgress = (progress: number): void => { + const currentProgress = + this.tokenCheckTransactionsStatus[this.currencyInfo.currencyCode] + const newProgress = progress + + if ( + // Only send event if we haven't completed sync + currentProgress < 1 && + // Avoid thrashing + (newProgress >= 1 || newProgress > currentProgress * 1.1) + ) { + this.tokenCheckTransactionsStatus[this.currencyInfo.currencyCode] = + newProgress + this.updateOnAddressesChecked() + } } - } - const handleScan = ({ - tx, - progress - }: { - tx: EdgeTransaction - progress: number - }): void => { - this.addTransaction(this.currencyInfo.currencyCode, tx) - this.onUpdateTransactions() - - // Progress the block-height if the message's height is greater than - // last poll for block-height. - if (this.walletLocalData.blockHeight < tx.blockHeight) { - this.onUpdateBlockHeight(tx.blockHeight) - } + const handleScan = ({ + tx, + progress + }: { + tx: EdgeTransaction + progress: number + }): void => { + this.addTransaction(this.currencyInfo.currencyCode, tx) + this.onUpdateTransactions() + + // Progress the block-height if the message's height is greater than + // last poll for block-height. + if (this.walletLocalData.blockHeight < tx.blockHeight) { + this.onUpdateBlockHeight(tx.blockHeight) + } - handleScanProgress(progress) - } + handleScanProgress(progress) + } - const scanners = [ - this.scanTransactionsFromFilscan(addressString, handleScan), - this.scanTransactionsFromFilfox(addressString, handleScan) - ] + const scanners = [ + this.scanTransactionsFromFilscan(addressString, handleScan), + this.scanTransactionsFromFilfox(addressString, handleScan) + ] - await Promise.all(scanners) + await Promise.all(scanners) - handleScanProgress(1) + handleScanProgress(1) + } catch (error) { + console.error(error) + throw error + } finally { + this.isScanning = false + } } async scanTransactionsFromFilfox( From 933b4742d31aae8842b888076bdbbd1e961d2d16 Mon Sep 17 00:00:00 2001 From: Samuel Holmes Date: Fri, 8 Sep 2023 14:41:30 -0700 Subject: [PATCH 3/6] Use new `lastAddressQueryHeight` data during sync The transaction list data doesn't load until `getTransactions` is called or a new transaction is saved. This means it's not suitable for checking the known txid from prior syncs. --- src/filecoin/FilecoinEngine.ts | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/filecoin/FilecoinEngine.ts b/src/filecoin/FilecoinEngine.ts index 5da43bbb5..926a3a2ff 100644 --- a/src/filecoin/FilecoinEngine.ts +++ b/src/filecoin/FilecoinEngine.ts @@ -301,6 +301,7 @@ export class FilecoinEngine extends CurrencyEngine< } async checkTransactions(): Promise { + // We shouldn't start scanning if scanning is already happening: if (this.isScanning) return try { this.isScanning = true @@ -348,8 +349,16 @@ export class FilecoinEngine extends CurrencyEngine< this.scanTransactionsFromFilfox(addressString, handleScan) ] + const startingNetworkHeight = this.walletLocalData.blockHeight + + // Run scanners: await Promise.all(scanners) + // Save the network height at the start of the scanning + this.walletLocalData.lastAddressQueryHeight = startingNetworkHeight + this.walletLocalDataDirty = true + + // Make sure the sync progress is 100% handleScanProgress(1) } catch (error) { console.error(error) @@ -381,14 +390,8 @@ export class FilecoinEngine extends CurrencyEngine< const messages = messagesResponse.messages for (const message of messages) { - const txid = message.cid - const idx = this.findTransaction(this.currencyInfo.currencyCode, txid) - - if (idx >= 0) { - // Exit early because we reached transaction history from previous - // check - return - } + // Exit when we reach a transaction we may already have saved + if (message.height < this.walletLocalData.lastAddressQueryHeight) return // Process message into a transaction const tx = this.filfoxMessageToEdgeTransaction(message) @@ -424,14 +427,8 @@ export class FilecoinEngine extends CurrencyEngine< const messages = messagesResponse.messages_by_account_id_list for (const message of messages) { - const txid = message.cid - const idx = this.findTransaction(this.currencyInfo.currencyCode, txid) - - if (idx >= 0) { - // Exit early because we reached transaction history from previous - // check - return - } + // Exit when we reach a transaction we may already have saved + if (message.height < this.walletLocalData.lastAddressQueryHeight) return // Process message into a transaction const tx = this.filscanMessageToEdgeTransaction(message) From ec219cab3b229a76c5cd1c843d40532308bc296a Mon Sep 17 00:00:00 2001 From: Samuel Holmes Date: Fri, 8 Sep 2023 14:42:03 -0700 Subject: [PATCH 4/6] Reset tokenCheckTransactionsStatus during engine start --- src/filecoin/FilecoinEngine.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/filecoin/FilecoinEngine.ts b/src/filecoin/FilecoinEngine.ts index 926a3a2ff..9a21f8587 100644 --- a/src/filecoin/FilecoinEngine.ts +++ b/src/filecoin/FilecoinEngine.ts @@ -87,9 +87,7 @@ export class FilecoinEngine extends CurrencyEngine< } initData(): void { - // Initialize walletLocalData: - // ... - + this.tokenCheckTransactionsStatus[this.currencyInfo.currencyCode] = 0 // Engine variables this.availableAttoFil = '0' } From 0fab2a27e61c9d8ce9ce1e9dd94a1c93f8d0c327 Mon Sep 17 00:00:00 2001 From: Samuel Holmes Date: Thu, 7 Sep 2023 16:05:14 -0700 Subject: [PATCH 5/6] Implement `getMessageDetails` endpoint in Filfox to include fee data --- src/filecoin/FilecoinEngine.ts | 28 +++++++++----- src/filecoin/Filfox.ts | 67 ++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/filecoin/FilecoinEngine.ts b/src/filecoin/FilecoinEngine.ts index 9a21f8587..0cec51e2b 100644 --- a/src/filecoin/FilecoinEngine.ts +++ b/src/filecoin/FilecoinEngine.ts @@ -34,7 +34,7 @@ import { FilecoinWalletOtherData, SafeFilecoinWalletInfo } from './filecoinTypes' -import { Filfox, FilfoxMessage } from './Filfox' +import { Filfox, FilfoxMessageDetailed } from './Filfox' import { Filscan, FilscanMessage } from './Filscan' import { RpcExtra } from './RpcExtra' @@ -392,7 +392,10 @@ export class FilecoinEngine extends CurrencyEngine< if (message.height < this.walletLocalData.lastAddressQueryHeight) return // Process message into a transaction - const tx = this.filfoxMessageToEdgeTransaction(message) + const messageDetails = await this.filfoxApi.getMessageDetails( + message.cid + ) + const tx = this.filfoxMessageToEdgeTransaction(messageDetails) // Calculate the progress const progress = @@ -441,14 +444,21 @@ export class FilecoinEngine extends CurrencyEngine< } filfoxMessageToEdgeTransaction = ( - message: FilfoxMessage + messageDetails: FilfoxMessageDetailed ): EdgeTransaction => { const addressString = this.address.toString() - let netNativeAmount = message.value + let netNativeAmount = messageDetails.value const ourReceiveAddresses = [] - const networkFee = '0' // TODO: calculate transaction fee from onchain gas fields - if (message.to !== addressString) { + // Get the fees paid + const networkFee = messageDetails.transfers + .filter( + transfer => + transfer.type === 'miner-fee' || transfer.type === 'burner-fee' + ) + .reduce((sum, transfer) => add(sum, transfer.value), '0') + + if (messageDetails.to !== addressString) { // check if tx is a spend netNativeAmount = `-${add(netNativeAmount, networkFee)}` } else { @@ -456,10 +466,10 @@ export class FilecoinEngine extends CurrencyEngine< } const edgeTransaction: EdgeTransaction = { - txid: message.cid, - date: message.timestamp, + txid: messageDetails.cid, + date: messageDetails.timestamp, currencyCode: this.currencyInfo.currencyCode, - blockHeight: message.height, + blockHeight: messageDetails.height, nativeAmount: netNativeAmount, isSend: netNativeAmount.startsWith('-'), networkFee, diff --git a/src/filecoin/Filfox.ts b/src/filecoin/Filfox.ts index db9bc3e6b..694cc34bb 100644 --- a/src/filecoin/Filfox.ts +++ b/src/filecoin/Filfox.ts @@ -48,6 +48,42 @@ export const asFilfoxMessage = asObject({ value: asString }) +export type FilfoxMessageDetailed = ReturnType +export const asFilfoxMessageDetailed = asObject({ + cid: asString, + height: asNumber, + timestamp: asNumber, + from: asString, + to: asString, + value: asString, + gasLimit: asNumber, + gasFeeCap: asString, + gasPremium: asString, + receipt: asObject({ + exitCode: asNumber, + return: asString, + gasUsed: asNumber + }), + baseFee: asString, + fee: asObject({ + baseFeeBurn: asString, + overEstimationBurn: asString, + minerPenalty: asString, + minerTip: asString, + refund: asString + }), + transfers: asArray( + asObject({ + from: asString, + fromId: asString, + to: asString, + toId: asString, + value: asString, + type: asString + }) + ) +}) + // // Messages // @@ -58,6 +94,15 @@ export const asFilfoxMessagesResult = asObject({ totalCount: asNumber }) +// +// Message Details +// + +export type FilfoxMessageDetailsResult = ReturnType< + typeof asFilfoxMessageDetailsResult +> +export const asFilfoxMessageDetailsResult = asFilfoxMessageDetailed + // ----------------------------------------------------------------------------- // Implementation // ----------------------------------------------------------------------------- @@ -98,4 +143,26 @@ export class Filfox { return responseBody } + + async getMessageDetails( + messageCid: string + ): Promise { + const response = await this.fetch(`${this.baseUrl}/message/${messageCid}`, { + method: 'GET', + headers: { + 'content-type': 'application/json' + } + }) + const responseText = await response.text() + const responseBody = asFilfoxEnvelope(asFilfoxMessageDetailsResult)( + responseText + ) + + if ('error' in responseBody) + throw new Error( + `Error response code ${responseBody.statusCode}: ${responseBody.message} ${responseBody.error}` + ) + + return responseBody + } } From 4d957098c8df8503fed50f5433e42cb39def50f5 Mon Sep 17 00:00:00 2001 From: Samuel Holmes Date: Thu, 7 Sep 2023 16:05:33 -0700 Subject: [PATCH 6/6] Disable Filscan querying in favor of using Filfox exclusively Filfox has a convenient API for retrieving fee info on a message. Filscan doesn't have an known API for getting the fee amounts without figuring out how to calculate the fee amounts from four or five fee related fields on a Filecoin message. --- src/filecoin/FilecoinEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/filecoin/FilecoinEngine.ts b/src/filecoin/FilecoinEngine.ts index 0cec51e2b..5dd990763 100644 --- a/src/filecoin/FilecoinEngine.ts +++ b/src/filecoin/FilecoinEngine.ts @@ -343,7 +343,7 @@ export class FilecoinEngine extends CurrencyEngine< } const scanners = [ - this.scanTransactionsFromFilscan(addressString, handleScan), + // this.scanTransactionsFromFilscan(addressString, handleScan), this.scanTransactionsFromFilfox(addressString, handleScan) ]