0xGasless Plugin

package.json

@@ -43,7 +43,7 @@
     "biggystring": "^4.1.3",
     "body-parser": "^1.19.0",
     "cleaner-config": "^0.1.10",
-    "cleaners": "^0.3.13",
+    "cleaners": "^0.3.17",
     "commander": "^6.1.0",
     "cors": "^2.8.5",
     "csv-stringify": "^6.2.0",

src/config.ts

@@ -8,8 +8,12 @@ export const asConfig = asObject({
   ),
   httpPort: asOptional(asNumber, 8008),
   bog: asOptional(asObject({ apiKey: asString }), { apiKey: '' }),
+
+  /** Only run specific appIds (e.g. edge, coinhub, etc) */
   soloAppIds: asOptional(asArray(asString), null),
+  /** Only run specific partnerIds (e.g. moonpay, paybis, etc) */
   soloPartnerIds: asOptional(asArray(asString), null),
+
   timeoutOverrideMins: asOptional(asNumber, 1200),
   cacheLookbackMonths: asOptional(asNumber, 24)
 })

src/partners/0xgasless.ts

@@ -0,0 +1,245 @@
+import {
+  asArray,
+  asEither,
+  asJSON,
+  asNull,
+  asNumber,
+  asObject,
+  asString,
+  asUnknown,
+  asValue
+} from 'cleaners'
+import URL from 'url-parse'
+
+import {
+  asStandardPluginParams,
+  EDGE_APP_START_DATE,
+  PartnerPlugin,
+  PluginParams,
+  PluginResult,
+  StandardTx,
+  Status
+} from '../types'
+import { datelog, retryFetch, smartIsoDateFromTimestamp, snooze } from '../util'
+
+const API_URL = 'https://api.0x.org/trade-analytics/gasless'
+const PLUGIN_START_DATE = '2024-05-05T00:00:00.000Z'
+/** Max fetch retries before bailing */
+const MAX_RETRIES = 5
+/**
+ * How far to rollback from the last successful query
+ * date when starting a new query
+ */
+const QUERY_LOOKBACK = 1000 * 60 * 60 * 24 * 30 // 30 Days
+/** Time period to query per loop */
+const QUERY_TIME_BLOCK_MS = QUERY_LOOKBACK
+
+export async function query0xGasless(
+  pluginParams: PluginParams
+): Promise<PluginResult> {
+  const { settings, apiKeys } = asStandardPluginParams(pluginParams)
+
+  if (apiKeys.apiKey == null) {
+    throw new Error('0xGasless: Missing 0xgasless API key')
+  }
+  const nowDate = new Date()
+  const now = nowDate.getTime()
+
+  let { latestIsoDate } = settings
+
+  if (latestIsoDate === EDGE_APP_START_DATE) {
+    latestIsoDate = new Date(PLUGIN_START_DATE).toISOString()
+  }
+
+  let lastCheckedTimestamp = new Date(latestIsoDate).getTime() - QUERY_LOOKBACK
+  if (lastCheckedTimestamp < 0) lastCheckedTimestamp = 0
+
+  const ssFormatTxs: StandardTx[] = []
+  let retry = 0
+
+  while (true) {
+    let latestBlockIsoDate = latestIsoDate
+    const startTimestamp = lastCheckedTimestamp
+    const endTimestamp = lastCheckedTimestamp + QUERY_TIME_BLOCK_MS
+
+    try {
+      let cursor: string | undefined
+
+      while (true) {
+        const urlObj = new URL(API_URL, true)
+
+        const queryParams: {
+          startTimestamp: string
+          endTimestamp: string
+          cursor?: string
+        } = {
+          // API expects seconds-based unix timestamps
+          startTimestamp: Math.floor(startTimestamp / 1000).toString(),
+          endTimestamp: Math.floor(endTimestamp / 1000).toString()
+        }
+        if (cursor != null) queryParams.cursor = cursor
+        urlObj.set('query', queryParams)
+
+        datelog(
+          `0xGasless Querying from:${new Date(
+            startTimestamp
+          ).toISOString()} to:${new Date(endTimestamp).toISOString()}`
+        )
+
+        const url = urlObj.href
+        const response = await retryFetch(url, {
+          headers: {
+            '0x-api-key': apiKeys.apiKey,
+            '0x-version': 'v2'
+          }
+        })
+        const responseJson = await response.text()
+        if (!response.ok) {
+          throw new Error(`${url} response ${response.status}: ${responseJson}`)
+        }
+        const responseBody = asGetGaslessTradesResponse(responseJson)
+
+        for (const rawTx of responseBody.trades) {
+          const standardTx = process0xGaslessTx(rawTx)
+
+          ssFormatTxs.push(standardTx)
+          if (standardTx.isoDate > latestBlockIsoDate) {
+            latestBlockIsoDate = standardTx.isoDate
+          }
+        }
+
+        datelog(`0xGasless ${responseBody.trades.length} trades processed`)
+
+        if (responseBody.nextCursor == null) {
+          datelog(`0xGasless No cursor from API`)
+          break
+        } else {
+          cursor = responseBody.nextCursor
+          datelog(`0xGasless Get nextCursor: ${cursor}`)
+        }
+      }
+
+      lastCheckedTimestamp = endTimestamp
+      latestIsoDate = latestBlockIsoDate
+      datelog(
+        `0xGasless endDate:${new Date(
+          lastCheckedTimestamp
+        ).toISOString()} latestIsoDate:${latestIsoDate}`
+      )
+      if (lastCheckedTimestamp > now) {
+        break
+      }
+      retry = 0
+    } catch (error) {
+      datelog(error)
+      // Retry a few times with time delay to prevent throttling
+      retry++
+      if (retry <= MAX_RETRIES) {
+        datelog(`Snoozing ${60 * retry}s`)
+        await snooze(60000 * retry)
+      } else {
+        // We can safely save our progress since we go from oldest to newest,
+        // and we don't update the lastIsoDate until the page is processed
+        // fully.
+        break
+      }
+    }
+
+    // Wait before next query, to prevent rate-limiting and thrashing
+    await snooze(1000)
+  }
+
+  const out = {
+    settings: { latestIsoDate },
+    transactions: ssFormatTxs
+  }
+  return out
+}
+
+export const zeroxgasless: PartnerPlugin = {
+  queryFunc: query0xGasless,
+  pluginName: '0xGasless',
+  pluginId: '0xgasless'
+}
+
+export function process0xGaslessTx(rawTx: unknown): StandardTx {
+  const trade = asGaslessTrade(rawTx)
+  const buySymbol = trade.tokens.find(t => t.address === trade.buyToken)?.symbol
+  const sellSymbol = trade.tokens.find(t => t.address === trade.sellToken)
+    ?.symbol
+
+  if (buySymbol == null || sellSymbol == null) {
+    throw new Error(
+      `Could not find buy or sell symbol for trade with txid ${trade.transactionHash}`
+    )
+  }
+
+  const {
+    isoDate: tradeIsoDate,
+    timestamp: tradeTimestamp
+  } = smartIsoDateFromTimestamp(trade.timestamp * 1000)
+
+  // If trade is 2 days or older, then it's finalized according to 0x
+  // documentation.
+  const status: Status =
+    tradeTimestamp + 2 * 24 * 60 * 60 * 1000 < Date.now()
+      ? 'complete'
+      : 'pending'
+
+  const standardTx: StandardTx = {
+    status,
+    orderId: trade.transactionHash,
+    depositTxid: trade.transactionHash,
+    depositAddress: undefined,
+    depositCurrency: sellSymbol,
+    depositAmount: Number(trade.sellAmount),
+    payoutTxid: trade.transactionHash,
+    payoutAddress: trade.taker ?? undefined,
+    payoutCurrency: buySymbol,
+    payoutAmount: Number(trade.buyAmount),
+    timestamp: tradeTimestamp,
+    isoDate: tradeIsoDate,
+    usdValue: parseFloat(trade.volumeUsd),
+    rawTx: trade
+  }
+
+  return standardTx
+}
+
+const asGetGaslessTradesResponse = asJSON(
+  asObject({
+    nextCursor: asEither(asString, asNull),
+    trades: asArray(asUnknown)
+  })
+)
+
+const asGaslessTrade = asObject({
+  appName: asString,
+  blockNumber: asString,
+  buyToken: asString,
+  buyAmount: asString,
+  chainId: asNumber,
+  // Fee data is not used.
+  // fees: {
+  //   "integratorFee": null,
+  //   "zeroExFee": null
+  // },
+  gasUsed: asString,
+  protocolVersion: asString,
+  sellToken: asString,
+  sellAmount: asString,
+  slippageBps: asEither(asString, asNull),
+  taker: asString,
+  timestamp: asNumber,
+  tokens: asArray(v => asGaslessTradeToken(v)),
+  transactionHash: asString,
+  volumeUsd: asString,
+  /** The 0x trade id */
+  zid: asString,
+  service: asValue('gasless')
+})
+
+const asGaslessTradeToken = asObject({
+  address: asString,
+  symbol: asString
+})

src/partners/paybis.ts

@@ -175,6 +175,7 @@ export async function queryPaybis(
 
   while (true) {
     const endTime = startTime + QUERY_TIME_BLOCK_MS
+    let latestBlockIsoDate = latestIsoDate
 
     try {
       let cursor: string | undefined
@@ -239,8 +240,8 @@ export async function queryPaybis(
             rawTx
           }
           ssFormatTxs.push(ssTx)
-          if (ssTx.isoDate > latestIsoDate) {
-            latestIsoDate = ssTx.isoDate
+          if (ssTx.isoDate > latestBlockIsoDate) {
+            latestBlockIsoDate = ssTx.isoDate
           }
         }
         if (cursor == null) {
@@ -252,6 +253,7 @@ export async function queryPaybis(
 
       const endDate = new Date(endTime)
       startTime = endTime
+      latestIsoDate = latestBlockIsoDate
       datelog(
         `Paybis endDate:${endDate.toISOString()} latestIsoDate:${latestIsoDate}`
       )

src/queryEngine.ts

@@ -3,6 +3,7 @@ import nano from 'nano'
 import { config } from './config'
 import { pagination } from './dbutils'
 import { initDbs } from './initDbs'
+import { zeroxgasless } from './partners/0xgasless'
 import { banxa } from './partners/banxa'
 import { bitaccess } from './partners/bitaccess'
 import { bitrefill } from './partners/bitrefill'
@@ -68,7 +69,8 @@ const plugins = [
   thorchain,
   transak,
   wyre,
-  xanpool
+  xanpool,
+  zeroxgasless
 ]
 const QUERY_FREQ_MS = 60 * 1000
 const MAX_CONCURRENT_QUERIES = 3

yarn.lock

@@ -2073,10 +2073,10 @@ cleaner-config@^0.1.10:
     minimist "^1.2.5"
     sucrase "^3.17.1"
 
-cleaners@^0.3.13:
-  version "0.3.13"
-  resolved "https://registry.npmjs.org/cleaners/-/cleaners-0.3.13.tgz"
-  integrity sha512-sCedc8LIXUhLmXT9rkkAToi9mjYhI7J/gKRWiF0Qw6eC0ymILHxq+vhuaKoKdcSWpYi2YqqwSlvNtD+92gf4pA==
+cleaners@^0.3.17:
+  version "0.3.17"
+  resolved "https://registry.yarnpkg.com/cleaners/-/cleaners-0.3.17.tgz#dae498f3d49b7e9364050402d2f4ad09abcd31ba"
+  integrity sha512-X5acjsLwJK+JEK5hv0Rve7G78+E6iYh1TzJZ40z7Yjrba0WhW6spTq28WgG9w+AK+YQIOHtQTrzaiuntMBBIwQ==
 
 cleaners@^0.3.8:
   version "0.3.16"