diff --git a/packages/api/src/common/types.ts b/packages/api/src/common/types.ts index eb37a7b7ae..743cd580ca 100644 --- a/packages/api/src/common/types.ts +++ b/packages/api/src/common/types.ts @@ -6,6 +6,7 @@ interface IPaginationFilterOptions { blockNumber?: number; address?: string; l1BatchNumber?: number; + minLiquidity?: number; } export interface IPaginationOptions extends NestIPaginationOptions { diff --git a/packages/api/src/token/token.controller.spec.ts b/packages/api/src/token/token.controller.spec.ts index aca8783e78..2b6cfe890f 100644 --- a/packages/api/src/token/token.controller.spec.ts +++ b/packages/api/src/token/token.controller.spec.ts @@ -52,7 +52,10 @@ describe("TokenController", () => { it("queries tokens with the specified options", async () => { await controller.getTokens(pagingOptions, 1000); expect(serviceMock.findAll).toHaveBeenCalledTimes(1); - expect(serviceMock.findAll).toHaveBeenCalledWith({ minLiquidity: 1000 }, { ...pagingOptions, route: "tokens" }); + expect(serviceMock.findAll).toHaveBeenCalledWith( + { minLiquidity: 1000 }, + { ...pagingOptions, filterOptions: { minLiquidity: 1000 }, route: "tokens" } + ); }); it("returns the tokens", async () => { diff --git a/packages/api/src/token/token.controller.ts b/packages/api/src/token/token.controller.ts index 44ea613065..660f18ae3f 100644 --- a/packages/api/src/token/token.controller.ts +++ b/packages/api/src/token/token.controller.ts @@ -47,6 +47,7 @@ export class TokenController { minLiquidity, }, { + filterOptions: { minLiquidity }, ...pagingOptions, route: entityName, } diff --git a/packages/api/src/token/token.service.ts b/packages/api/src/token/token.service.ts index c4dd045655..4adafbc315 100644 --- a/packages/api/src/token/token.service.ts +++ b/packages/api/src/token/token.service.ts @@ -1,7 +1,8 @@ import { Injectable } from "@nestjs/common"; import { InjectRepository } from "@nestjs/typeorm"; import { Repository, FindOptionsSelect, MoreThanOrEqual } from "typeorm"; -import { Pagination, IPaginationOptions } from "nestjs-typeorm-paginate"; +import { Pagination } from "nestjs-typeorm-paginate"; +import { IPaginationOptions } from "../common/types"; import { paginate } from "../common/utils"; import { Token, ETH_TOKEN } from "./token.entity"; diff --git a/packages/app/src/composables/useTokenLibrary.ts b/packages/app/src/composables/useTokenLibrary.ts index 9c92820b45..f6e71044cd 100644 --- a/packages/app/src/composables/useTokenLibrary.ts +++ b/packages/app/src/composables/useTokenLibrary.ts @@ -7,10 +7,20 @@ import useContext, { type Context } from "@/composables/useContext"; const retrieveTokens = useMemoize( async (context: Context): Promise => { - const tokensResponse = await $fetch>( - `${context.currentNetwork.value.apiUrl}/tokens?minLiquidity=0&limit=100` - ); - return tokensResponse.items; + const tokens = []; + let page = 1; + let hasMore = true; + + while (hasMore) { + const tokensResponse = await $fetch>( + `${context.currentNetwork.value.apiUrl}/tokens?minLiquidity=0&limit=100&page=${page}` + ); + tokens.push(...tokensResponse.items); + page++; + hasMore = tokensResponse.meta.totalPages > tokensResponse.meta.currentPage; + } + + return tokens; }, { getKey(context: Context) { diff --git a/packages/app/src/locales/en.json b/packages/app/src/locales/en.json index f11d13509f..3f6a3c4810 100644 --- a/packages/app/src/locales/en.json +++ b/packages/app/src/locales/en.json @@ -583,6 +583,7 @@ "tokenListView": { "title": "Token List", "heading": "Tokens", + "offChainDataPoweredBy": "Off-chain data powered by", "table": { "tokenName": "Token Name", "price": "Price", diff --git a/packages/app/src/locales/uk.json b/packages/app/src/locales/uk.json index 579c69d370..47ad5b8c56 100644 --- a/packages/app/src/locales/uk.json +++ b/packages/app/src/locales/uk.json @@ -325,6 +325,7 @@ "tokenListView": { "title": "Список Токенів", "heading": "Токени", + "offChainDataPoweredBy": "Off-chain дані взяті з", "table": { "tokenName": "Назва Токена", "price": "Ціна", diff --git a/packages/app/src/views/TokensView.vue b/packages/app/src/views/TokensView.vue index 8d4cb8f0ab..7cedbb96b6 100644 --- a/packages/app/src/views/TokensView.vue +++ b/packages/app/src/views/TokensView.vue @@ -4,7 +4,13 @@ -

{{ t("tokenListView.heading") }}

+
+

{{ t("tokenListView.heading") }}

+
+ {{ t("tokenListView.offChainDataPoweredBy") }}{{ " " }} + CoinGecko API +
+
{{ t("failedRequest") }} @@ -51,4 +57,16 @@ getTokens(); .tokens-container { @apply mt-8; } + +.tokens-header { + @apply flex justify-between items-end gap-4; + + .coingecko-attribution { + @apply mr-1 text-gray-300; + + a { + @apply text-blue-100; + } + } +} diff --git a/packages/app/tests/composables/useTokenLibrary.spec.ts b/packages/app/tests/composables/useTokenLibrary.spec.ts index c6eca5bb79..1e7c734e16 100644 --- a/packages/app/tests/composables/useTokenLibrary.spec.ts +++ b/packages/app/tests/composables/useTokenLibrary.spec.ts @@ -13,16 +13,40 @@ vi.mock("ohmyfetch", async () => { const mod = await vi.importActual("ohmyfetch"); return { ...mod, - $fetch: vi.fn().mockResolvedValue([ - { - decimals: 18, - iconURL: "https://icon.url", - l1Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - l2Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - name: "Ether", - symbol: "ETH", - } as Api.Response.Token, - ]), + $fetch: vi + .fn() + .mockResolvedValueOnce({ + items: [ + { + decimals: 18, + iconURL: "https://icon.url", + l1Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + l2Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + name: "Ether", + symbol: "ETH", + } as Api.Response.Token, + ], + meta: { + totalPages: 2, + currentPage: 1, + }, + }) + .mockResolvedValueOnce({ + items: [ + { + decimals: 18, + iconURL: "https://icon2.url", + l1Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeef", + l2Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeef", + name: "Ether2", + symbol: "ETH2", + } as Api.Response.Token, + ], + meta: { + totalPages: 2, + currentPage: 2, + }, + }), }; }); @@ -31,7 +55,9 @@ describe("useTokenLibrary:", () => { const { getTokens } = useTokenLibrary(); await getTokens(); await getTokens(); - expect($fetch).toHaveBeenCalledTimes(1); + await getTokens(); + await getTokens(); + expect($fetch).toHaveBeenCalledTimes(2); }); it("sets isRequestPending to true when request is pending", async () => { const { isRequestPending, getTokens } = useTokenLibrary(); diff --git a/packages/worker/.env.example b/packages/worker/.env.example index eb575ece89..7402762a37 100644 --- a/packages/worker/.env.example +++ b/packages/worker/.env.example @@ -32,7 +32,10 @@ DISABLE_BLOCKS_REVERT=false ENABLE_TOKEN_OFFCHAIN_DATA_SAVER=false UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL=86400000 -TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER=0 +SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER=coingecko FROM_BLOCK=0 TO_BLOCK= + +COINGECKO_IS_PRO_PLAN=false +COINGECKO_API_KEY= diff --git a/packages/worker/src/app.module.ts b/packages/worker/src/app.module.ts index 43ce3900fb..7fc4504e81 100644 --- a/packages/worker/src/app.module.ts +++ b/packages/worker/src/app.module.ts @@ -1,8 +1,8 @@ import { Module, Logger } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; -import { ConfigModule } from "@nestjs/config"; +import { ConfigModule, ConfigService } from "@nestjs/config"; import { EventEmitterModule } from "@nestjs/event-emitter"; -import { HttpModule } from "@nestjs/axios"; +import { HttpModule, HttpService } from "@nestjs/axios"; import { PrometheusModule } from "@willsoto/nestjs-prometheus"; import config from "./config"; import { HealthModule } from "./health/health.module"; @@ -18,7 +18,8 @@ import { BalanceService, BalancesCleanerService } from "./balance"; import { TransferService } from "./transfer/transfer.service"; import { TokenService } from "./token/token.service"; import { TokenOffChainDataProvider } from "./token/tokenOffChainData/tokenOffChainDataProvider.abstract"; -import { PortalsFiTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider"; +import { CoingeckoTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider"; +import { PortalsFiTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider"; import { TokenOffChainDataSaverService } from "./token/tokenOffChainData/tokenOffChainDataSaver.service"; import { CounterModule } from "./counter/counter.module"; import { @@ -100,7 +101,16 @@ import { UnitOfWorkModule } from "./unitOfWork"; TokenService, { provide: TokenOffChainDataProvider, - useClass: PortalsFiTokenOffChainDataProvider, + useFactory: (configService: ConfigService, httpService: HttpService) => { + const selectedProvider = configService.get("tokens.selectedTokenOffChainDataProvider"); + switch (selectedProvider) { + case "portalsFi": + return new PortalsFiTokenOffChainDataProvider(httpService); + default: + return new CoingeckoTokenOffChainDataProvider(configService, httpService); + } + }, + inject: [ConfigService, HttpService], }, TokenOffChainDataSaverService, BatchRepository, diff --git a/packages/worker/src/config.spec.ts b/packages/worker/src/config.spec.ts index 2b5ae3909a..89c3d438f4 100644 --- a/packages/worker/src/config.spec.ts +++ b/packages/worker/src/config.spec.ts @@ -45,7 +45,11 @@ describe("config", () => { tokens: { enableTokenOffChainDataSaver: false, updateTokenOffChainDataInterval: 86_400_000, - tokenOffChainDataMinLiquidityFilter: 0, + tokenOffChainDataProviders: ["coingecko", "portalsFi"], + selectedTokenOffChainDataProvider: "coingecko", + coingecko: { + isProPlan: false, + }, }, metrics: { collectDbConnectionPoolMetricsInterval: 10000, diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index bf14f74486..53e0ed8258 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -21,9 +21,11 @@ export default () => { DISABLE_BLOCKS_REVERT, ENABLE_TOKEN_OFFCHAIN_DATA_SAVER, UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL, - TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER, + SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER, FROM_BLOCK, TO_BLOCK, + COINGECKO_IS_PRO_PLAN, + COINGECKO_API_KEY, } = process.env; return { @@ -59,7 +61,12 @@ export default () => { tokens: { enableTokenOffChainDataSaver: ENABLE_TOKEN_OFFCHAIN_DATA_SAVER === "true", updateTokenOffChainDataInterval: parseInt(UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL, 10) || 86_400_000, - tokenOffChainDataMinLiquidityFilter: parseInt(TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER, 10) || 0, + tokenOffChainDataProviders: ["coingecko", "portalsFi"], + selectedTokenOffChainDataProvider: SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER || "coingecko", + coingecko: { + isProPlan: COINGECKO_IS_PRO_PLAN === "true", + apiKey: COINGECKO_API_KEY, + }, }, metrics: { collectDbConnectionPoolMetricsInterval: parseInt(COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL, 10) || 10000, diff --git a/packages/worker/src/repositories/token.repository.spec.ts b/packages/worker/src/repositories/token.repository.spec.ts index 8d5955961c..f0be5707eb 100644 --- a/packages/worker/src/repositories/token.repository.spec.ts +++ b/packages/worker/src/repositories/token.repository.spec.ts @@ -174,6 +174,61 @@ describe("TokenRepository", () => { }); describe("updateTokenOffChainData", () => { + it("throws error when no l1Address or l2Address provided", async () => { + const updatedAt = new Date(); + await expect( + repository.updateTokenOffChainData({ + liquidity: 1000000, + usdPrice: 55.89037747, + updatedAt, + }) + ).rejects.toThrowError("l1Address or l2Address must be provided"); + }); + + it("updates token offchain data using l1Address when provided", async () => { + const updatedAt = new Date(); + await repository.updateTokenOffChainData({ + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + liquidity: 1000000, + usdPrice: 55.89037747, + updatedAt, + }); + + expect(entityManagerMock.update).toBeCalledWith( + Token, + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + }, + { + liquidity: 1000000, + usdPrice: 55.89037747, + offChainDataUpdatedAt: updatedAt, + } + ); + }); + + it("updates token offchain data using l2Address when provided", async () => { + const updatedAt = new Date(); + await repository.updateTokenOffChainData({ + l2Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + liquidity: 1000000, + usdPrice: 55.89037747, + updatedAt, + }); + + expect(entityManagerMock.update).toBeCalledWith( + Token, + { + l2Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + }, + { + liquidity: 1000000, + usdPrice: 55.89037747, + offChainDataUpdatedAt: updatedAt, + } + ); + }); + it("updates token offchain data when iconURL is not provided", async () => { const updatedAt = new Date(); await repository.updateTokenOffChainData({ diff --git a/packages/worker/src/repositories/token.repository.ts b/packages/worker/src/repositories/token.repository.ts index 5b9e701096..dd5de197ed 100644 --- a/packages/worker/src/repositories/token.repository.ts +++ b/packages/worker/src/repositories/token.repository.ts @@ -63,22 +63,27 @@ export class TokenRepository extends BaseRepository { public async updateTokenOffChainData({ l1Address, + l2Address, liquidity, usdPrice, updatedAt, iconURL, }: { - l1Address: string; - liquidity: number; - usdPrice: number; - updatedAt: Date; + l1Address?: string; + l2Address?: string; + liquidity?: number; + usdPrice?: number; + updatedAt?: Date; iconURL?: string; }): Promise { + if (!l1Address && !l2Address) { + throw new Error("l1Address or l2Address must be provided"); + } const transactionManager = this.unitOfWork.getTransactionManager(); await transactionManager.update( this.entityTarget, { - l1Address, + ...(l1Address ? { l1Address } : { l2Address }), }, { liquidity, diff --git a/packages/worker/src/token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider.spec.ts b/packages/worker/src/token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider.spec.ts new file mode 100644 index 0000000000..eb64860589 --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider.spec.ts @@ -0,0 +1,315 @@ +import { Test } from "@nestjs/testing"; +import { mock } from "jest-mock-extended"; +import { Logger } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { HttpService } from "@nestjs/axios"; +import { AxiosResponse, AxiosError } from "axios"; +import { setTimeout } from "timers/promises"; +import * as rxjs from "rxjs"; +import { CoingeckoTokenOffChainDataProvider } from "./coingeckoTokenOffChainDataProvider"; + +const bridgedTokens = ["someAddress", "address1"]; +const providerTokensListResponse = [ + { + id: "ethereum", + platforms: {}, + }, + { + id: "token1", + platforms: { + ethereum: "address1", + }, + }, + { + id: "token2", + platforms: { + somePlatform: "address22", + zksync: "address2", + }, + }, + { + id: "token3", + platforms: { + somePlatform: "address3", + }, + }, +]; + +jest.useFakeTimers().setSystemTime(new Date("2023-01-01T02:00:00.000Z")); + +jest.mock("timers/promises", () => ({ + setTimeout: jest.fn().mockResolvedValue(null), +})); + +describe("CoingeckoTokenOffChainDataProvider", () => { + let provider: CoingeckoTokenOffChainDataProvider; + let configServiceMock: ConfigService; + let httpServiceMock: HttpService; + + beforeEach(async () => { + configServiceMock = mock({ + get: jest.fn().mockReturnValueOnce(true).mockReturnValueOnce("apiKey"), + }); + httpServiceMock = mock(); + const module = await Test.createTestingModule({ + providers: [ + CoingeckoTokenOffChainDataProvider, + { + provide: ConfigService, + useValue: configServiceMock, + }, + { + provide: HttpService, + useValue: httpServiceMock, + }, + ], + }).compile(); + module.useLogger(mock()); + + provider = module.get(CoingeckoTokenOffChainDataProvider); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe("getTokensOffChainData", () => { + let pipeMock = jest.fn(); + + beforeEach(() => { + pipeMock = jest.fn(); + jest.spyOn(httpServiceMock, "get").mockReturnValue({ + pipe: pipeMock, + } as unknown as rxjs.Observable); + jest.spyOn(rxjs, "catchError").mockImplementation((callback) => callback as any); + }); + + it("uses correct API url and API key for pro plan", async () => { + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + status: 404, + }, + } as AxiosError); + }); + + await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledWith( + "https://pro-api.coingecko.com/api/v3/coins/list?include_platform=true&x_cg_pro_api_key=apiKey" + ); + }); + + it("uses correct API url and API key for demo plan", async () => { + const module = await Test.createTestingModule({ + providers: [ + CoingeckoTokenOffChainDataProvider, + { + provide: ConfigService, + useValue: mock({ + get: jest.fn().mockReturnValueOnce(false).mockReturnValueOnce("apiKey"), + }), + }, + { + provide: HttpService, + useValue: httpServiceMock, + }, + ], + }).compile(); + module.useLogger(mock()); + const providerWithDemoPlan = module.get(CoingeckoTokenOffChainDataProvider); + + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + status: 404, + }, + } as AxiosError); + }); + + await providerWithDemoPlan.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledWith( + "https://api.coingecko.com/api/v3/coins/list?include_platform=true&x_cg_demo_api_key=apiKey" + ); + }); + + it("returns empty array when fetching tokens list constantly fails", async () => { + pipeMock.mockImplementation((callback) => { + callback({ + stack: "error stack", + } as AxiosError); + }); + + const tokens = await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(tokens).toEqual([]); + }); + + it("retries for 5 times each time doubling timeout when fetching tokens list constantly fails", async () => { + pipeMock.mockImplementation((callback) => { + callback({ + stack: "error stack", + response: { + data: "response data", + status: 500, + }, + } as AxiosError); + }); + + await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledTimes(6); + expect(setTimeout).toBeCalledTimes(5); + }); + + describe("when provider rate limit is reached", () => { + describe("when server provides rate limit reset date", () => { + it("retries API call after waiting for rate limit to reset if reset Date is in the future", async () => { + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + headers: { + "x-ratelimit-reset": "2023-01-01 02:02:00 +0000", + } as any, + status: 429, + }, + } as AxiosError); + }); + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + status: 404, + }, + } as AxiosError); + }); + + await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledTimes(2); + expect(setTimeout).toBeCalledTimes(1); + expect(setTimeout).toBeCalledWith(121000); + }); + + it("retries API call after immediately if reset Date is not in the future", async () => { + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + headers: { + "x-ratelimit-reset": "2023-01-01 01:59:00 +0000", + } as any, + status: 429, + }, + } as AxiosError); + }); + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + status: 404, + }, + } as AxiosError); + }); + + await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledTimes(2); + expect(setTimeout).toBeCalledTimes(1); + expect(setTimeout).toBeCalledWith(0); + }); + }); + + describe("when server does not provide rate limit reset date", () => { + it("retries API call after waiting for 61 seconds", async () => { + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + headers: {}, + status: 429, + }, + } as AxiosError); + }); + pipeMock.mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + status: 404, + }, + } as AxiosError); + }); + + await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledTimes(2); + expect(setTimeout).toBeCalledTimes(1); + expect(setTimeout).toBeCalledWith(61000); + }); + }); + }); + + it("fetches offchain tokens data and returns filtered tokens list", async () => { + pipeMock + .mockReturnValueOnce( + new rxjs.Observable((subscriber) => { + subscriber.next({ + data: providerTokensListResponse, + }); + }) + ) + .mockReturnValueOnce( + new rxjs.Observable((subscriber) => { + subscriber.next({ + data: [ + { + id: "ethereum", + market_cap: 100, + current_price: 10, + image: "http://ethereum.img", + }, + { + id: "token1", + market_cap: 101, + current_price: 11, + image: "http://token1.img", + }, + { + id: "token2", + market_cap: 102, + current_price: 12, + image: "http://token2.img", + }, + ], + }); + }) + ); + + const tokens = await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(httpServiceMock.get).toBeCalledTimes(2); + expect(httpServiceMock.get).toBeCalledWith( + "https://pro-api.coingecko.com/api/v3/coins/list?include_platform=true&x_cg_pro_api_key=apiKey" + ); + expect(httpServiceMock.get).toBeCalledWith( + "https://pro-api.coingecko.com/api/v3/coins/markets?vs_currency=usd&ids=ethereum%2Ctoken1%2Ctoken2&per_page=3&page=1&locale=en&x_cg_pro_api_key=apiKey" + ); + expect(tokens).toEqual([ + { + l1Address: "0x0000000000000000000000000000000000000000", + liquidity: 100, + usdPrice: 10, + iconURL: "http://ethereum.img", + }, + { + l1Address: "address1", + liquidity: 101, + usdPrice: 11, + iconURL: "http://token1.img", + }, + { + l2Address: "address2", + liquidity: 102, + usdPrice: 12, + iconURL: "http://token2.img", + }, + ]); + }); + }); +}); diff --git a/packages/worker/src/token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider.ts b/packages/worker/src/token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider.ts new file mode 100644 index 0000000000..5acdc9f8bd --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider.ts @@ -0,0 +1,204 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { HttpService } from "@nestjs/axios"; +import { ConfigService } from "@nestjs/config"; +import { AxiosError } from "axios"; +import { setTimeout } from "timers/promises"; +import { catchError, firstValueFrom } from "rxjs"; +import { utils } from "zksync-web3"; +import { TokenOffChainDataProvider, ITokenOffChainData } from "../../tokenOffChainDataProvider.abstract"; + +const API_NUMBER_OF_TOKENS_PER_REQUEST = 250; +const API_INITIAL_RETRY_TIMEOUT = 5000; +const API_RETRY_ATTEMPTS = 5; + +interface ITokenListItemProviderResponse { + id: string; + platforms: Record; +} + +interface ITokenMarketDataProviderResponse { + id: string; + image?: string; + current_price?: number; + market_cap?: number; +} + +class ProviderResponseError extends Error { + constructor(message: string, public readonly status: number, public readonly rateLimitResetDate?: Date) { + super(message); + } +} + +@Injectable() +export class CoingeckoTokenOffChainDataProvider implements TokenOffChainDataProvider { + private readonly logger: Logger; + private readonly isProPlan: boolean; + private readonly apiKey: string; + private readonly apiUrl: string; + + constructor(configService: ConfigService, private readonly httpService: HttpService) { + this.logger = new Logger(CoingeckoTokenOffChainDataProvider.name); + this.isProPlan = configService.get("tokens.coingecko.isProPlan"); + this.apiKey = configService.get("tokens.coingecko.apiKey"); + this.apiUrl = this.isProPlan ? "https://pro-api.coingecko.com/api/v3" : "https://api.coingecko.com/api/v3"; + } + + public async getTokensOffChainData({ + bridgedTokensToInclude, + }: { + bridgedTokensToInclude: string[]; + }): Promise { + const tokensList = await this.getTokensList(); + // Include ETH, all zksync L2 tokens and bridged tokens + const supportedTokens = tokensList.filter( + (token) => + token.id === "ethereum" || + token.platforms.zksync || + bridgedTokensToInclude.find((bridgetTokenAddress) => bridgetTokenAddress === token.platforms.ethereum) + ); + + const tokensOffChainData: ITokenOffChainData[] = []; + let tokenIdsPerRequest = []; + for (let i = 0; i < supportedTokens.length; i++) { + tokenIdsPerRequest.push(supportedTokens[i].id); + if (tokenIdsPerRequest.length === API_NUMBER_OF_TOKENS_PER_REQUEST || i === supportedTokens.length - 1) { + const tokensMarkedData = await this.getTokensMarketData(tokenIdsPerRequest); + tokensOffChainData.push( + ...tokensMarkedData.map((tokenMarketData) => { + const token = supportedTokens.find((t) => t.id === tokenMarketData.id); + return { + l1Address: token.id === "ethereum" ? utils.ETH_ADDRESS : token.platforms.ethereum, + l2Address: token.platforms.zksync, + liquidity: tokenMarketData.market_cap, + usdPrice: tokenMarketData.current_price, + iconURL: tokenMarketData.image, + }; + }) + ); + tokenIdsPerRequest = []; + } + } + return tokensOffChainData; + } + + private getTokensMarketData(tokenIds: string[]) { + return this.makeApiRequestRetryable({ + path: "/coins/markets", + query: { + vs_currency: "usd", + ids: tokenIds.join(","), + per_page: tokenIds.length.toString(), + page: "1", + locale: "en", + }, + }); + } + + private async getTokensList() { + const list = await this.makeApiRequestRetryable({ + path: "/coins/list", + query: { + include_platform: "true", + }, + }); + if (!list) { + return []; + } + return list + .filter((item) => item.id === "ethereum" || item.platforms.zksync || item.platforms.ethereum) + .map((item) => ({ + ...item, + platforms: { + // use substring(0, 42) to fix some instances when after address there is some additional text + zksync: item.platforms.zksync?.substring(0, 42), + ethereum: item.platforms.ethereum?.substring(0, 42), + }, + })); + } + + private async makeApiRequestRetryable({ + path, + query, + retryAttempt = 0, + retryTimeout = API_INITIAL_RETRY_TIMEOUT, + }: { + path: string; + query?: Record; + retryAttempt?: number; + retryTimeout?: number; + }): Promise { + try { + return await this.makeApiRequest(path, query); + } catch (err) { + if (err.status === 404) { + return null; + } + if (err.status === 429) { + const rateLimitResetIn = err.rateLimitResetDate.getTime() - new Date().getTime(); + await setTimeout(rateLimitResetIn >= 0 ? rateLimitResetIn + 1000 : 0); + return this.makeApiRequestRetryable({ + path, + query, + }); + } + if (retryAttempt >= API_RETRY_ATTEMPTS) { + this.logger.error({ + message: `Failed to fetch data at ${path} from coingecko after ${retryAttempt} retries`, + provider: CoingeckoTokenOffChainDataProvider.name, + }); + return null; + } + await setTimeout(retryTimeout); + return this.makeApiRequestRetryable({ + path, + query, + retryAttempt: retryAttempt + 1, + retryTimeout: retryTimeout * 2, + }); + } + } + + private async makeApiRequest(path: string, query?: Record): Promise { + const queryString = new URLSearchParams({ + ...query, + ...(this.isProPlan + ? { + x_cg_pro_api_key: this.apiKey, + } + : { + x_cg_demo_api_key: this.apiKey, + }), + }).toString(); + + const { data } = await firstValueFrom<{ data: T }>( + this.httpService.get(`${this.apiUrl}${path}?${queryString}`).pipe( + catchError((error: AxiosError) => { + if (error.response?.status === 429) { + const rateLimitReset = error.response.headers["x-ratelimit-reset"]; + // use specified reset date or 60 seconds by default + const rateLimitResetDate = rateLimitReset + ? new Date(rateLimitReset) + : new Date(new Date().getTime() + 60000); + this.logger.debug({ + message: `Reached coingecko rate limit, reset at ${rateLimitResetDate}`, + stack: error.stack, + status: error.response.status, + response: error.response.data, + provider: CoingeckoTokenOffChainDataProvider.name, + }); + throw new ProviderResponseError(error.message, error.response.status, rateLimitResetDate); + } + this.logger.error({ + message: `Failed to fetch data at ${path} from coingecko`, + stack: error.stack, + status: error.response?.status, + response: error.response?.data, + provider: CoingeckoTokenOffChainDataProvider.name, + }); + throw new ProviderResponseError(error.message, error.response?.status); + }) + ) + ); + return data; + } +} diff --git a/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.spec.ts b/packages/worker/src/token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider.spec.ts similarity index 84% rename from packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.spec.ts rename to packages/worker/src/token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider.spec.ts index b79df56888..1b93928d4d 100644 --- a/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.spec.ts +++ b/packages/worker/src/token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider.spec.ts @@ -7,10 +7,10 @@ import { setTimeout } from "timers/promises"; import * as rxjs from "rxjs"; import { PortalsFiTokenOffChainDataProvider } from "./portalsFiTokenOffChainDataProvider"; -const MIN_TOKENS_LIQUIDITY_FILTER = 0; const TOKENS_INFO_API_URL = "https://api.portals.fi/v2/tokens"; const TOKENS_INFO_API_QUERY = `networks=ethereum&limit=250&sortBy=liquidity&sortDirection=desc`; +const bridgedTokens = ["address1", "address2", "address3"]; const providerTokensResponse = [ { address: "address1", @@ -29,6 +29,11 @@ const providerTokensResponse = [ liquidity: 3000000, price: 10.7678787, }, + { + address: "unknown-token-address", + liquidity: 0, + price: 0, + }, ]; jest.mock("timers/promises", () => ({ @@ -77,7 +82,13 @@ describe("PortalsFiTokenOffChainDataProvider", () => { } as AxiosError); }); - const tokens = await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + const tokens = await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); + expect(tokens).toEqual([]); + }); + + it("returns empty array when no bridgedTokensToInclude provided", async () => { + const tokens = await provider.getTokensOffChainData({ bridgedTokensToInclude: [] }); + expect(httpServiceMock.get).not.toBeCalled(); expect(tokens).toEqual([]); }); @@ -92,7 +103,7 @@ describe("PortalsFiTokenOffChainDataProvider", () => { } as AxiosError); }); - await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); expect(httpServiceMock.get).toBeCalledTimes(6); expect(setTimeout).toBeCalledTimes(5); }); @@ -120,7 +131,7 @@ describe("PortalsFiTokenOffChainDataProvider", () => { }) ); - const tokens = await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + const tokens = await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); expect(httpServiceMock.get).toBeCalledTimes(2); expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0`); expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=1`); @@ -140,25 +151,6 @@ describe("PortalsFiTokenOffChainDataProvider", () => { ]); }); - it("includes minLiquidity filter when provided minLiquidity filter > 0", async () => { - pipeMock.mockReturnValueOnce( - new rxjs.Observable((subscriber) => { - subscriber.next({ - data: { - more: false, - tokens: [providerTokensResponse[0]], - }, - }); - }) - ); - - await provider.getTokensOffChainData(1000000); - expect(httpServiceMock.get).toBeCalledTimes(1); - expect(httpServiceMock.get).toBeCalledWith( - `${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0&minLiquidity=1000000` - ); - }); - it("retries when provider API call fails", async () => { pipeMock .mockImplementationOnce((callback) => { @@ -181,7 +173,7 @@ describe("PortalsFiTokenOffChainDataProvider", () => { }) ); - const tokens = await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + const tokens = await provider.getTokensOffChainData({ bridgedTokensToInclude: bridgedTokens }); expect(httpServiceMock.get).toBeCalledTimes(2); expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0`); expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0`); diff --git a/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.ts b/packages/worker/src/token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider.ts similarity index 83% rename from packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.ts rename to packages/worker/src/token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider.ts index 98ff62281a..ed61d85357 100644 --- a/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.ts +++ b/packages/worker/src/token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider.ts @@ -3,7 +3,7 @@ import { HttpService } from "@nestjs/axios"; import { AxiosError } from "axios"; import { setTimeout } from "timers/promises"; import { catchError, firstValueFrom } from "rxjs"; -import { TokenOffChainDataProvider, ITokenOffChainData } from "../tokenOffChainDataProvider.abstract"; +import { TokenOffChainDataProvider, ITokenOffChainData } from "../../tokenOffChainDataProvider.abstract"; const TOKENS_INFO_API_URL = "https://api.portals.fi/v2/tokens"; const API_INITIAL_RETRY_TIMEOUT = 5000; @@ -35,34 +35,43 @@ export class PortalsFiTokenOffChainDataProvider implements TokenOffChainDataProv this.logger = new Logger(PortalsFiTokenOffChainDataProvider.name); } - public async getTokensOffChainData(minLiquidity: number): Promise { + public async getTokensOffChainData({ + bridgedTokensToInclude, + }: { + bridgedTokensToInclude: string[]; + }): Promise { let page = 0; let hasMore = true; const tokens = []; + // This provider only supports bridged tokens + if (!bridgedTokensToInclude.length) { + return []; + } + while (hasMore) { - const tokensInfoPage = await this.getTokensOffChainDataPageRetryable({ page, minLiquidity }); + const tokensInfoPage = await this.getTokensOffChainDataPageRetryable({ page }); tokens.push(...tokensInfoPage.tokens); page++; hasMore = tokensInfoPage.hasMore; } - return tokens; + return tokens.filter((token) => + bridgedTokensToInclude.find((bridgetTokenAddress) => bridgetTokenAddress === token.l1Address) + ); } private async getTokensOffChainDataPageRetryable({ page, - minLiquidity, retryAttempt = 0, retryTimeout = API_INITIAL_RETRY_TIMEOUT, }: { page: number; - minLiquidity: number; retryAttempt?: number; retryTimeout?: number; }): Promise { try { - return await this.getTokensOffChainDataPage({ page, minLiquidity }); + return await this.getTokensOffChainDataPage({ page }); } catch { if (retryAttempt >= API_RETRY_ATTEMPTS) { this.logger.error({ @@ -77,29 +86,19 @@ export class PortalsFiTokenOffChainDataProvider implements TokenOffChainDataProv await setTimeout(retryTimeout); return this.getTokensOffChainDataPageRetryable({ page, - minLiquidity, retryAttempt: retryAttempt + 1, retryTimeout: retryTimeout * 2, }); } } - private async getTokensOffChainDataPage({ - page, - minLiquidity, - }: { - page: number; - minLiquidity: number; - }): Promise { + private async getTokensOffChainDataPage({ page }: { page: number }): Promise { const query = { networks: "ethereum", limit: "250", sortBy: "liquidity", sortDirection: "desc", page: page.toString(), - ...(minLiquidity && { - minLiquidity: minLiquidity.toString(), - }), }; const queryString = new URLSearchParams(query).toString(); diff --git a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts index ff99d4cec7..6d169c848e 100644 --- a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts +++ b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts @@ -1,10 +1,11 @@ export interface ITokenOffChainData { - l1Address: string; - liquidity: number; - usdPrice: number; + l1Address?: string; + l2Address?: string; + liquidity?: number; + usdPrice?: number; iconURL?: string; } export abstract class TokenOffChainDataProvider { - abstract getTokensOffChainData: (minLiquidity: number) => Promise; + abstract getTokensOffChainData: (settings: { bridgedTokensToInclude: string[] }) => Promise; } diff --git a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts index 2e810e3a86..f2c97456bf 100644 --- a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts +++ b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts @@ -20,9 +20,9 @@ jest.mock("../../utils/waitFor"); describe("TokenOffChainDataSaverService", () => { const OFFCHAIN_DATA_UPDATE_INTERVAL = 86_400_000; - const MIN_LIQUIDITY_FILTER = 1000000; const tokenOffChainDataMock = { - l1Address: "address", + l1Address: "l1Address", + l2Address: "l2Address", liquidity: 100000, usdPrice: 12.6789, iconURL: "http://icon.com", @@ -46,11 +46,7 @@ describe("TokenOffChainDataSaverService", () => { tokenRepositoryMock, tokenOffChainDataProviderMock, mock({ - get: jest - .fn() - .mockImplementation((key) => - key === "tokens.updateTokenOffChainDataInterval" ? OFFCHAIN_DATA_UPDATE_INTERVAL : MIN_LIQUIDITY_FILTER - ), + get: jest.fn().mockReturnValue(OFFCHAIN_DATA_UPDATE_INTERVAL), }) ); }); @@ -91,31 +87,21 @@ describe("TokenOffChainDataSaverService", () => { expect(tokenOffChainDataProviderMock.getTokensOffChainData).not.toBeCalled(); }); - it("does not update offchain data when there are no bridged token atm and waits for the next update", async () => { - tokenOffChainDataSaverService.start(); - await tokenOffChainDataSaverService.stop(); - - const [conditionPredicate, waitTime] = (waitFor as jest.Mock).mock.calls[0]; - expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(1); - expect(waitFor).toBeCalledTimes(1); - expect(conditionPredicate()).toBeTruthy(); - expect(waitTime).toBe(OFFCHAIN_DATA_UPDATE_INTERVAL); - expect(tokenRepositoryMock.getBridgedTokens).toBeCalledTimes(1); - expect(tokenOffChainDataProviderMock.getTokensOffChainData).not.toBeCalled(); - }); - - it("updates offchain data when data is too old and there are bridged tokens to update", async () => { + it("updates offchain data when data is too old", async () => { const lastUpdatedAt = new Date("2022-01-01T01:00:00.000Z"); jest.spyOn(tokenRepositoryMock, "getOffChainDataLastUpdatedAt").mockResolvedValueOnce(lastUpdatedAt); - jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "address" } as Token]); + jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "l1Address" } as Token]); tokenOffChainDataSaverService.start(); await tokenOffChainDataSaverService.stop(); - expect(tokenOffChainDataProviderMock.getTokensOffChainData).toBeCalledWith(MIN_LIQUIDITY_FILTER); + expect(tokenOffChainDataProviderMock.getTokensOffChainData).toBeCalledWith({ + bridgedTokensToInclude: ["l1Address"], + }); expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledTimes(1); expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledWith({ - l1Address: "address", + l1Address: "l1Address", + l2Address: "l2Address", liquidity: 100000, usdPrice: 12.6789, updatedAt: new Date(), @@ -123,16 +109,19 @@ describe("TokenOffChainDataSaverService", () => { }); }); - it("updates offchain data when data was never updated and there are bridged tokens to update", async () => { - jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "address" } as Token]); + it("updates offchain data when data was never updated", async () => { + jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "l1Address" } as Token]); tokenOffChainDataSaverService.start(); await tokenOffChainDataSaverService.stop(); - expect(tokenOffChainDataProviderMock.getTokensOffChainData).toBeCalledWith(MIN_LIQUIDITY_FILTER); + expect(tokenOffChainDataProviderMock.getTokensOffChainData).toBeCalledWith({ + bridgedTokensToInclude: ["l1Address"], + }); expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledTimes(1); expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledWith({ - l1Address: "address", + l1Address: "l1Address", + l2Address: "l2Address", liquidity: 100000, usdPrice: 12.6789, updatedAt: new Date(), @@ -141,7 +130,7 @@ describe("TokenOffChainDataSaverService", () => { }); it("waits for specified timeout or worker stoppage after offchain data update", async () => { - jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "address" } as Token]); + jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "l1Address" } as Token]); tokenOffChainDataSaverService.start(); await tokenOffChainDataSaverService.stop(); diff --git a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts index 5084c27a6f..6e600b2293 100644 --- a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts +++ b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts @@ -10,7 +10,6 @@ const UPDATE_TOKENS_BATCH_SIZE = 100; @Injectable() export class TokenOffChainDataSaverService extends Worker { private readonly updateTokenOffChainDataInterval: number; - private readonly tokenOffChainDataMinLiquidityFilter: number; private readonly logger: Logger; public constructor( @@ -20,7 +19,6 @@ export class TokenOffChainDataSaverService extends Worker { ) { super(); this.updateTokenOffChainDataInterval = configService.get("tokens.updateTokenOffChainDataInterval"); - this.tokenOffChainDataMinLiquidityFilter = configService.get("tokens.tokenOffChainDataMinLiquidityFilter"); this.logger = new Logger(TokenOffChainDataSaverService.name); } @@ -37,37 +35,33 @@ export class TokenOffChainDataSaverService extends Worker { if (!nextUpdateTimeout) { const bridgedTokens = await this.tokenRepository.getBridgedTokens(); - if (bridgedTokens.length) { - const tokensInfo = await this.tokenOffChainDataProvider.getTokensOffChainData( - this.tokenOffChainDataMinLiquidityFilter - ); - const tokensToUpdate = tokensInfo.filter((token) => - bridgedTokens.find((t) => t.l1Address === token.l1Address) - ); - const updatedAt = new Date(); + const tokensToUpdate = await this.tokenOffChainDataProvider.getTokensOffChainData({ + bridgedTokensToInclude: bridgedTokens.map((t) => t.l1Address), + }); + const updatedAt = new Date(); - let updateTokensTasks = []; - for (let i = 0; i < tokensToUpdate.length; i++) { - updateTokensTasks.push( - this.tokenRepository.updateTokenOffChainData({ - l1Address: tokensToUpdate[i].l1Address, - liquidity: tokensToUpdate[i].liquidity, - usdPrice: tokensToUpdate[i].usdPrice, - updatedAt, - iconURL: tokensToUpdate[i].iconURL, - }) - ); - if (updateTokensTasks.length === UPDATE_TOKENS_BATCH_SIZE || i === tokensToUpdate.length - 1) { - await Promise.all(updateTokensTasks); - updateTokensTasks = []; - } + let updateTokensTasks = []; + for (let i = 0; i < tokensToUpdate.length; i++) { + updateTokensTasks.push( + this.tokenRepository.updateTokenOffChainData({ + l1Address: tokensToUpdate[i].l1Address, + l2Address: tokensToUpdate[i].l2Address, + liquidity: tokensToUpdate[i].liquidity, + usdPrice: tokensToUpdate[i].usdPrice, + updatedAt, + iconURL: tokensToUpdate[i].iconURL, + }) + ); + if (updateTokensTasks.length === UPDATE_TOKENS_BATCH_SIZE || i === tokensToUpdate.length - 1) { + await Promise.all(updateTokensTasks); + updateTokensTasks = []; } - - this.logger.log("Updated tokens offchain data", { - totalTokensUpdated: tokensToUpdate.length, - }); } + this.logger.log("Updated tokens offchain data", { + totalTokensUpdated: tokensToUpdate.length, + }); + nextUpdateTimeout = this.updateTokenOffChainDataInterval; } } catch (err) {