From b2cdf17326714675f81bcb7757bdf9faf61f82b5 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Mon, 6 Jan 2025 14:50:28 -0800 Subject: [PATCH 1/7] Add comments to config definition --- src/config.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config.ts b/src/config.ts index 9bfc4083..92b904f4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,8 +8,12 @@ export const asConfig = asObject({ ), httpPort: asOptional(asNumber, 8008), bog: asOptional(asObject({ apiKey: asString }), { apiKey: '' }), + + /** Only run specific appIds (e.g. edge, coinhub, etc) */ soloAppIds: asOptional(asArray(asString), null), + /** Only run specific partnerIds (e.g. moonpay, paybis, etc) */ soloPartnerIds: asOptional(asArray(asString), null), + timeoutOverrideMins: asOptional(asNumber, 1200), cacheLookbackMonths: asOptional(asNumber, 24) }) From aa136dc5818752e4abfcc839f58ff0c20b72396d Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Mon, 6 Jan 2025 16:30:48 -0800 Subject: [PATCH 2/7] Upgrade cleaners@^0.3.17 This upgrade gives us more information about the raw value's type when a cleaner fails. Very useful. --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index f0d89874..9947064b 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "biggystring": "^4.1.3", "body-parser": "^1.19.0", "cleaner-config": "^0.1.10", - "cleaners": "^0.3.13", + "cleaners": "^0.3.17", "commander": "^6.1.0", "cors": "^2.8.5", "csv-stringify": "^6.2.0", diff --git a/yarn.lock b/yarn.lock index 3b13a007..4dbe53d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2073,10 +2073,10 @@ cleaner-config@^0.1.10: minimist "^1.2.5" sucrase "^3.17.1" -cleaners@^0.3.13: - version "0.3.13" - resolved "https://registry.npmjs.org/cleaners/-/cleaners-0.3.13.tgz" - integrity sha512-sCedc8LIXUhLmXT9rkkAToi9mjYhI7J/gKRWiF0Qw6eC0ymILHxq+vhuaKoKdcSWpYi2YqqwSlvNtD+92gf4pA== +cleaners@^0.3.17: + version "0.3.17" + resolved "https://registry.yarnpkg.com/cleaners/-/cleaners-0.3.17.tgz#dae498f3d49b7e9364050402d2f4ad09abcd31ba" + integrity sha512-X5acjsLwJK+JEK5hv0Rve7G78+E6iYh1TzJZ40z7Yjrba0WhW6spTq28WgG9w+AK+YQIOHtQTrzaiuntMBBIwQ== cleaners@^0.3.8: version "0.3.16" From fd391251319ac8e154cb8b0cda70454b9205cca7 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Mon, 6 Jan 2025 17:24:43 -0800 Subject: [PATCH 3/7] Add 0xGasless plugin --- src/partners/0xgasless.ts | 250 ++++++++++++++++++++++++++++++++++++++ src/queryEngine.ts | 4 +- 2 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 src/partners/0xgasless.ts diff --git a/src/partners/0xgasless.ts b/src/partners/0xgasless.ts new file mode 100644 index 00000000..fa5ea774 --- /dev/null +++ b/src/partners/0xgasless.ts @@ -0,0 +1,250 @@ +import { + asArray, + asEither, + asJSON, + asNull, + asNumber, + asObject, + asString, + asValue +} from 'cleaners' +import URL from 'url-parse' + +import { + asStandardPluginParams, + EDGE_APP_START_DATE, + PartnerPlugin, + PluginParams, + PluginResult, + StandardTx, + Status +} from '../types' +import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util' + +const API_URL = 'https://api.0x.org/trade-analytics/gasless' +const PLUGIN_START_DATE = '2024-05-05T00:00:00.000Z' +/** Max fetch retries before bailing */ +const MAX_RETRIES = 5 +/** + * How far to rollback from the last successful query + * date when starting a new query + */ +const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 30 // 30 Days +/** Time period to query per loop */ +const QUERY_TIME_BLOCK_MS = QUERY_LOOKBACK + +type PartnerStatuses = + | 'other' + | 'created' + | 'completed' + | 'cancelled' + | 'payment_error' + | 'rejected' +const statusMap: { [key in PartnerStatuses]: Status } = { + created: 'pending', + cancelled: 'refunded', + payment_error: 'refunded', + completed: 'complete', + rejected: 'refunded', + other: 'other' +} + +export async function query0xGasless( + pluginParams: PluginParams +): Promise { + const { settings, apiKeys } = asStandardPluginParams(pluginParams) + + if (apiKeys.apiKey == null) { + throw new Error('0xGasless: Missing 0xgasless API key') + } + const nowDate = new Date() + const now = nowDate.getTime() + + let { latestIsoDate } = settings + + if (latestIsoDate === EDGE_APP_START_DATE) { + latestIsoDate = new Date(PLUGIN_START_DATE).toISOString() + } + + let lastCheckedTimestamp = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK + if (lastCheckedTimestamp < 0) lastCheckedTimestamp = 0 + + const ssFormatTxs: StandardTx[] = [] + let retry = 0 + + while (true) { + const startTimestamp = lastCheckedTimestamp + const endTimestamp = lastCheckedTimestamp + QUERY_TIME_BLOCK_MS + + try { + let cursor: string | undefined + + while (true) { + const urlObj = new URL(API_URL, true) + + const queryParams: { + startTimestamp: string + endTimestamp: string + cursor?: string + } = { + // API expects seconds-based unix timestamps + startTimestamp: Math.floor(startTimestamp / 1000).toString(), + endTimestamp: Math.floor(endTimestamp / 1000).toString() + } + if (cursor != null) queryParams.cursor = cursor + urlObj.set('query', queryParams) + + datelog( + `0xGasless Querying from:${new Date( + startTimestamp + ).toISOString()} to:${new Date(endTimestamp).toISOString()}` + ) + + const url = urlObj.href + const response = await retryFetch(url, { + headers: { + '0x-api-key': apiKeys.apiKey, + '0x-version': 'v2' + } + }) + const responseJson = await response.text() + if (!response.ok) { + throw new Error(`${url} response ${response.status}: ${responseJson}`) + } + const responseBody = asGetGaslessTradesResponse(responseJson) + + for (const trade of responseBody.trades) { + const buySymbol = trade.tokens.find(t => t.address === trade.buyToken) + ?.symbol + const sellSymbol = trade.tokens.find( + t => t.address === trade.sellToken + )?.symbol + + if (buySymbol == null || sellSymbol == null) { + throw new Error( + `Could not find buy or sell symbol for trade ${trade.zid}; txid: ${trade.transactionHash}` + ) + } + + const { + isoDate: tradeIsoDate, + timestamp: tradeTimestamp + } = smartIsoDateFromTimestamp(trade.timestamp * 1000) + + // If trade is 2 days or older, then it's finalized according to 0x + // documentation. + const status: Status = + tradeTimestamp + 2 * 24 * 60 * 60 * 1000 < now + ? 'complete' + : 'pending' + + const ssTx: StandardTx = { + status, + orderId: trade.zid, + depositTxid: trade.transactionHash, + depositAddress: undefined, + depositCurrency: sellSymbol, + depositAmount: Number(trade.sellAmount), + payoutTxid: trade.transactionHash, + payoutAddress: trade.taker ?? undefined, + payoutCurrency: buySymbol, + payoutAmount: Number(trade.buyAmount), + timestamp: tradeTimestamp, + isoDate: tradeIsoDate, + usdValue: parseFloat(trade.volumeUsd), + rawTx: trade + } + ssFormatTxs.push(ssTx) + if (ssTx.isoDate > latestIsoDate) { + latestIsoDate = ssTx.isoDate + } + } + + datelog(`0xGasless ${responseBody.trades.length} trades processed`) + + if (responseBody.nextCursor == null) { + datelog(`0xGasless No cursor from API`) + break + } else { + cursor = responseBody.nextCursor + datelog(`0xGasless Get nextCursor: ${cursor}`) + } + } + + lastCheckedTimestamp = endTimestamp + datelog( + `0xGasless endDate:${new Date( + lastCheckedTimestamp + ).toISOString()} latestIsoDate:${latestIsoDate}` + ) + if (lastCheckedTimestamp > now) { + break + } + retry = 0 + } catch (error) { + datelog(error) + // Retry a few times with time delay to prevent throttling + retry++ + if (retry <= MAX_RETRIES) { + datelog(`Snoozing ${60 * retry}s`) + await snooze(60000 * retry) + } else { + // We can't safely save our progress since we go from newest to oldest. + throw error + } + } + + // Wait before next query, to prevent rate-limiting and thrashing + await snooze(1000) + } + + const out = { + settings: { latestIsoDate }, + transactions: ssFormatTxs + } + return out +} + +export const zeroxgasless: PartnerPlugin = { + queryFunc: query0xGasless, + pluginName: '0xGasless', + pluginId: '0xgasless' +} + +const asGetGaslessTradesResponse = asJSON( + asObject({ + nextCursor: asEither(asString, asNull), + trades: asArray(v => asGaslessTrade(v)) + }) +) + +const asGaslessTrade = asObject({ + appName: asString, + blockNumber: asString, + buyToken: asString, + buyAmount: asString, + chainId: asNumber, + // Fee data is not used. + // fees: { + // "integratorFee": null, + // "zeroExFee": null + // }, + gasUsed: asString, + protocolVersion: asString, + sellToken: asString, + sellAmount: asString, + slippageBps: asEither(asString, asNull), + taker: asString, + timestamp: asNumber, + tokens: asArray(v => asGaslessTradeToken(v)), + transactionHash: asString, + volumeUsd: asString, + /** The 0x trade id */ + zid: asString, + service: asValue('gasless') +}) + +const asGaslessTradeToken = asObject({ + address: asString, + symbol: asString +}) diff --git a/src/queryEngine.ts b/src/queryEngine.ts index 1bcc80e6..13b34baf 100644 --- a/src/queryEngine.ts +++ b/src/queryEngine.ts @@ -3,6 +3,7 @@ import nano from 'nano' import { config } from './config' import { pagination } from './dbutils' import { initDbs } from './initDbs' +import { zeroxgasless } from './partners/0xgasless' import { banxa } from './partners/banxa' import { bitaccess } from './partners/bitaccess' import { bitrefill } from './partners/bitrefill' @@ -68,7 +69,8 @@ const plugins = [ thorchain, transak, wyre, - xanpool + xanpool, + zeroxgasless ] const QUERY_FREQ_MS = 60 * 1000 const MAX_CONCURRENT_QUERIES = 3 From 308c4fa6780c5b8002218a996751f1562006bba6 Mon Sep 17 00:00:00 2001 From: Paul V Puey Date: Wed, 15 Jan 2025 12:44:16 -0800 Subject: [PATCH 4/7] fixup! Add 0xGasless plugin --- src/partners/0xgasless.ts | 10 ++++++---- src/partners/paybis.ts | 6 ++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/partners/0xgasless.ts b/src/partners/0xgasless.ts index fa5ea774..06e5f9f0 100644 --- a/src/partners/0xgasless.ts +++ b/src/partners/0xgasless.ts @@ -73,6 +73,7 @@ export async function query0xGasless( let retry = 0 while (true) { + let latestBlockIsoDate = latestIsoDate const startTimestamp = lastCheckedTimestamp const endTimestamp = lastCheckedTimestamp + QUERY_TIME_BLOCK_MS @@ -155,8 +156,8 @@ export async function query0xGasless( rawTx: trade } ssFormatTxs.push(ssTx) - if (ssTx.isoDate > latestIsoDate) { - latestIsoDate = ssTx.isoDate + if (ssTx.isoDate > latestBlockIsoDate) { + latestBlockIsoDate = ssTx.isoDate } } @@ -172,6 +173,7 @@ export async function query0xGasless( } lastCheckedTimestamp = endTimestamp + latestIsoDate = latestBlockIsoDate datelog( `0xGasless endDate:${new Date( lastCheckedTimestamp @@ -189,8 +191,8 @@ export async function query0xGasless( datelog(`Snoozing ${60 * retry}s`) await snooze(60000 * retry) } else { - // We can't safely save our progress since we go from newest to oldest. - throw error + // We can safely save our progress since we go from oldest to newest. + break } } diff --git a/src/partners/paybis.ts b/src/partners/paybis.ts index 7e921b53..ae91d3d2 100644 --- a/src/partners/paybis.ts +++ b/src/partners/paybis.ts @@ -175,6 +175,7 @@ export async function queryPaybis( while (true) { const endTime = startTime + QUERY_TIME_BLOCK_MS + let latestBlockIsoDate = latestIsoDate try { let cursor: string | undefined @@ -239,8 +240,8 @@ export async function queryPaybis( rawTx } ssFormatTxs.push(ssTx) - if (ssTx.isoDate > latestIsoDate) { - latestIsoDate = ssTx.isoDate + if (ssTx.isoDate > latestBlockIsoDate) { + latestBlockIsoDate = ssTx.isoDate } } if (cursor == null) { @@ -252,6 +253,7 @@ export async function queryPaybis( const endDate = new Date(endTime) startTime = endTime + latestIsoDate = latestBlockIsoDate datelog( `Paybis endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}` ) From 3bdae8bfb5f8128b4b4cc730e177caa5337f2a41 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Thu, 16 Jan 2025 12:10:15 -0800 Subject: [PATCH 5/7] fixup! Add 0xGasless plugin --- src/partners/0xgasless.ts | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/partners/0xgasless.ts b/src/partners/0xgasless.ts index 06e5f9f0..670e975a 100644 --- a/src/partners/0xgasless.ts +++ b/src/partners/0xgasless.ts @@ -33,22 +33,6 @@ const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 30 // 30 Days /** Time period to query per loop */ const QUERY_TIME_BLOCK_MS = QUERY_LOOKBACK -type PartnerStatuses = - | 'other' - | 'created' - | 'completed' - | 'cancelled' - | 'payment_error' - | 'rejected' -const statusMap: { [key in PartnerStatuses]: Status } = { - created: 'pending', - cancelled: 'refunded', - payment_error: 'refunded', - completed: 'complete', - rejected: 'refunded', - other: 'other' -} - export async function query0xGasless( pluginParams: PluginParams ): Promise { @@ -123,7 +107,7 @@ export async function query0xGasless( if (buySymbol == null || sellSymbol == null) { throw new Error( - `Could not find buy or sell symbol for trade ${trade.zid}; txid: ${trade.transactionHash}` + `Could not find buy or sell symbol for trade with txid ${trade.transactionHash}` ) } @@ -141,7 +125,7 @@ export async function query0xGasless( const ssTx: StandardTx = { status, - orderId: trade.zid, + orderId: trade.transactionHash, depositTxid: trade.transactionHash, depositAddress: undefined, depositCurrency: sellSymbol, From af2fdfc361ddb5c4736f2a5a84e0170fd1aa3630 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Thu, 16 Jan 2025 13:26:24 -0800 Subject: [PATCH 6/7] fixup! Add 0xGasless plugin --- src/partners/0xgasless.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/partners/0xgasless.ts b/src/partners/0xgasless.ts index 670e975a..5c9e8847 100644 --- a/src/partners/0xgasless.ts +++ b/src/partners/0xgasless.ts @@ -175,7 +175,9 @@ export async function query0xGasless( datelog(`Snoozing ${60 * retry}s`) await snooze(60000 * retry) } else { - // We can safely save our progress since we go from oldest to newest. + // We can safely save our progress since we go from oldest to newest, + // and we don't update the lastIsoDate until the page is processed + // fully. break } } From b9f6a87debe82e373e12bbd9b96a401c93897e95 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Thu, 16 Jan 2025 13:29:23 -0800 Subject: [PATCH 7/7] fixup! Add 0xGasless plugin --- src/partners/0xgasless.ts | 95 +++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/src/partners/0xgasless.ts b/src/partners/0xgasless.ts index 5c9e8847..434f08fd 100644 --- a/src/partners/0xgasless.ts +++ b/src/partners/0xgasless.ts @@ -6,6 +6,7 @@ import { asNumber, asObject, asString, + asUnknown, asValue } from 'cleaners' import URL from 'url-parse' @@ -98,50 +99,12 @@ export async function query0xGasless( } const responseBody = asGetGaslessTradesResponse(responseJson) - for (const trade of responseBody.trades) { - const buySymbol = trade.tokens.find(t => t.address === trade.buyToken) - ?.symbol - const sellSymbol = trade.tokens.find( - t => t.address === trade.sellToken - )?.symbol - - if (buySymbol == null || sellSymbol == null) { - throw new Error( - `Could not find buy or sell symbol for trade with txid ${trade.transactionHash}` - ) - } + for (const rawTx of responseBody.trades) { + const standardTx = process0xGaslessTx(rawTx) - const { - isoDate: tradeIsoDate, - timestamp: tradeTimestamp - } = smartIsoDateFromTimestamp(trade.timestamp * 1000) - - // If trade is 2 days or older, then it's finalized according to 0x - // documentation. - const status: Status = - tradeTimestamp + 2 * 24 * 60 * 60 * 1000 < now - ? 'complete' - : 'pending' - - const ssTx: StandardTx = { - status, - orderId: trade.transactionHash, - depositTxid: trade.transactionHash, - depositAddress: undefined, - depositCurrency: sellSymbol, - depositAmount: Number(trade.sellAmount), - payoutTxid: trade.transactionHash, - payoutAddress: trade.taker ?? undefined, - payoutCurrency: buySymbol, - payoutAmount: Number(trade.buyAmount), - timestamp: tradeTimestamp, - isoDate: tradeIsoDate, - usdValue: parseFloat(trade.volumeUsd), - rawTx: trade - } - ssFormatTxs.push(ssTx) - if (ssTx.isoDate > latestBlockIsoDate) { - latestBlockIsoDate = ssTx.isoDate + ssFormatTxs.push(standardTx) + if (standardTx.isoDate > latestBlockIsoDate) { + latestBlockIsoDate = standardTx.isoDate } } @@ -199,10 +162,54 @@ export const zeroxgasless: PartnerPlugin = { pluginId: '0xgasless' } +export function process0xGaslessTx(rawTx: unknown): StandardTx { + const trade = asGaslessTrade(rawTx) + const buySymbol = trade.tokens.find(t => t.address === trade.buyToken)?.symbol + const sellSymbol = trade.tokens.find(t => t.address === trade.sellToken) + ?.symbol + + if (buySymbol == null || sellSymbol == null) { + throw new Error( + `Could not find buy or sell symbol for trade with txid ${trade.transactionHash}` + ) + } + + const { + isoDate: tradeIsoDate, + timestamp: tradeTimestamp + } = smartIsoDateFromTimestamp(trade.timestamp * 1000) + + // If trade is 2 days or older, then it's finalized according to 0x + // documentation. + const status: Status = + tradeTimestamp + 2 * 24 * 60 * 60 * 1000 < Date.now() + ? 'complete' + : 'pending' + + const standardTx: StandardTx = { + status, + orderId: trade.transactionHash, + depositTxid: trade.transactionHash, + depositAddress: undefined, + depositCurrency: sellSymbol, + depositAmount: Number(trade.sellAmount), + payoutTxid: trade.transactionHash, + payoutAddress: trade.taker ?? undefined, + payoutCurrency: buySymbol, + payoutAmount: Number(trade.buyAmount), + timestamp: tradeTimestamp, + isoDate: tradeIsoDate, + usdValue: parseFloat(trade.volumeUsd), + rawTx: trade + } + + return standardTx +} + const asGetGaslessTradesResponse = asJSON( asObject({ nextCursor: asEither(asString, asNull), - trades: asArray(v => asGaslessTrade(v)) + trades: asArray(asUnknown) }) )