feat: add coingecko token offchain data provider (#102)

# What ❔ - add coingecko tokens off-chain data provider ## Why ❔ - to support both L1 and L2 zksync tokens ## Checklist   - [+ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ +] Tests for the changes have been added / updated.
matter-labs · Feb 15, 2024 · 8717940 · 8717940
1 parent 60780c6
commit 8717940
Show file tree

Hide file tree

Showing 22 changed files with 775 additions and 135 deletions.
diff --git a/packages/api/src/common/types.ts b/packages/api/src/common/types.ts
@@ -6,6 +6,7 @@ interface IPaginationFilterOptions {
   blockNumber?: number;
   address?: string;
   l1BatchNumber?: number;
+  minLiquidity?: number;
 }
 
 export interface IPaginationOptions<CustomMetaType = IPaginationMeta> extends NestIPaginationOptions<CustomMetaType> {

diff --git a/packages/api/src/token/token.controller.spec.ts b/packages/api/src/token/token.controller.spec.ts
@@ -52,7 +52,10 @@ describe("TokenController", () => {
     it("queries tokens with the specified options", async () => {
       await controller.getTokens(pagingOptions, 1000);
       expect(serviceMock.findAll).toHaveBeenCalledTimes(1);
-      expect(serviceMock.findAll).toHaveBeenCalledWith({ minLiquidity: 1000 }, { ...pagingOptions, route: "tokens" });
+      expect(serviceMock.findAll).toHaveBeenCalledWith(
+        { minLiquidity: 1000 },
+        { ...pagingOptions, filterOptions: { minLiquidity: 1000 }, route: "tokens" }
+      );
     });
 
     it("returns the tokens", async () => {

diff --git a/packages/api/src/token/token.controller.ts b/packages/api/src/token/token.controller.ts
@@ -47,6 +47,7 @@ export class TokenController {
         minLiquidity,
       },
       {
+        filterOptions: { minLiquidity },
         ...pagingOptions,
         route: entityName,
       }

diff --git a/packages/api/src/token/token.service.ts b/packages/api/src/token/token.service.ts
@@ -1,7 +1,8 @@
 import { Injectable } from "@nestjs/common";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Repository, FindOptionsSelect, MoreThanOrEqual } from "typeorm";
-import { Pagination, IPaginationOptions } from "nestjs-typeorm-paginate";
+import { Pagination } from "nestjs-typeorm-paginate";
+import { IPaginationOptions } from "../common/types";
 import { paginate } from "../common/utils";
 import { Token, ETH_TOKEN } from "./token.entity";
 

diff --git a/packages/app/src/composables/useTokenLibrary.ts b/packages/app/src/composables/useTokenLibrary.ts
@@ -7,10 +7,20 @@ import useContext, { type Context } from "@/composables/useContext";
 
 const retrieveTokens = useMemoize(
   async (context: Context): Promise<Api.Response.Token[]> => {
-    const tokensResponse = await $fetch<Api.Response.Collection<Api.Response.Token>>(
-      `${context.currentNetwork.value.apiUrl}/tokens?minLiquidity=0&limit=100`
-    );
-    return tokensResponse.items;
+    const tokens = [];
+    let page = 1;
+    let hasMore = true;
+
+    while (hasMore) {
+      const tokensResponse = await $fetch<Api.Response.Collection<Api.Response.Token>>(
+        `${context.currentNetwork.value.apiUrl}/tokens?minLiquidity=0&limit=100&page=${page}`
+      );
+      tokens.push(...tokensResponse.items);
+      page++;
+      hasMore = tokensResponse.meta.totalPages > tokensResponse.meta.currentPage;
+    }
+
+    return tokens;
   },
   {
     getKey(context: Context) {

diff --git a/packages/app/src/locales/en.json b/packages/app/src/locales/en.json
@@ -583,6 +583,7 @@
     "tokenListView": {
         "title": "Token List",
         "heading": "Tokens",
+        "offChainDataPoweredBy": "Off-chain data powered by",
         "table": {
             "tokenName": "Token Name",
             "price": "Price",

diff --git a/packages/app/src/locales/uk.json b/packages/app/src/locales/uk.json
@@ -325,6 +325,7 @@
     "tokenListView": {
         "title": "Список Токенів",
         "heading": "Токени",
+        "offChainDataPoweredBy": "Off-chain дані взяті з",
         "table": {
             "tokenName": "Назва Токена",
             "price": "Ціна",

diff --git a/packages/app/src/views/TokensView.vue b/packages/app/src/views/TokensView.vue
@@ -4,7 +4,13 @@
       <Breadcrumbs :items="breadcrumbItems" />
       <SearchForm class="search-form" />
     </div>
-    <h1>{{ t("tokenListView.heading") }}</h1>
+    <div class="tokens-header">
+      <h1>{{ t("tokenListView.heading") }}</h1>
+      <div v-if="tokens[0]?.iconURL" class="coingecko-attribution">
+        <span>{{ t("tokenListView.offChainDataPoweredBy") }}{{ " " }}</span>
+        <a href="https://www.coingecko.com/en/api" target="_blank">CoinGecko API</a>
+      </div>
+    </div>
     <div class="tokens-container">
       <span v-if="isTokensFailed" class="error-message">
         {{ t("failedRequest") }}
@@ -51,4 +57,16 @@ getTokens();
 .tokens-container {
   @apply mt-8;
 }
+
+.tokens-header {
+  @apply flex justify-between items-end gap-4;
+
+  .coingecko-attribution {
+    @apply mr-1 text-gray-300;
+
+    a {
+      @apply text-blue-100;
+    }
+  }
+}
 </style>
diff --git a/packages/app/tests/composables/useTokenLibrary.spec.ts b/packages/app/tests/composables/useTokenLibrary.spec.ts
@@ -13,16 +13,40 @@ vi.mock("ohmyfetch", async () => {
   const mod = await vi.importActual<typeof import("ohmyfetch")>("ohmyfetch");
   return {
     ...mod,
-    $fetch: vi.fn().mockResolvedValue([
-      {
-        decimals: 18,
-        iconURL: "https://icon.url",
-        l1Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
-        l2Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
-        name: "Ether",
-        symbol: "ETH",
-      } as Api.Response.Token,
-    ]),
+    $fetch: vi
+      .fn()
+      .mockResolvedValueOnce({
+        items: [
+          {
+            decimals: 18,
+            iconURL: "https://icon.url",
+            l1Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+            l2Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee",
+            name: "Ether",
+            symbol: "ETH",
+          } as Api.Response.Token,
+        ],
+        meta: {
+          totalPages: 2,
+          currentPage: 1,
+        },
+      })
+      .mockResolvedValueOnce({
+        items: [
+          {
+            decimals: 18,
+            iconURL: "https://icon2.url",
+            l1Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeef",
+            l2Address: "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeef",
+            name: "Ether2",
+            symbol: "ETH2",
+          } as Api.Response.Token,
+        ],
+        meta: {
+          totalPages: 2,
+          currentPage: 2,
+        },
+      }),
   };
 });
 
@@ -31,7 +55,9 @@ describe("useTokenLibrary:", () => {
     const { getTokens } = useTokenLibrary();
     await getTokens();
     await getTokens();
-    expect($fetch).toHaveBeenCalledTimes(1);
+    await getTokens();
+    await getTokens();
+    expect($fetch).toHaveBeenCalledTimes(2);
   });
   it("sets isRequestPending to true when request is pending", async () => {
     const { isRequestPending, getTokens } = useTokenLibrary();

diff --git a/packages/worker/.env.example b/packages/worker/.env.example
@@ -32,7 +32,10 @@ DISABLE_BLOCKS_REVERT=false
 
 ENABLE_TOKEN_OFFCHAIN_DATA_SAVER=false
 UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL=86400000
-TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER=0
+SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER=coingecko
 
 FROM_BLOCK=0
 TO_BLOCK=
+
+COINGECKO_IS_PRO_PLAN=false
+COINGECKO_API_KEY=
diff --git a/packages/worker/src/app.module.ts b/packages/worker/src/app.module.ts
@@ -1,8 +1,8 @@
 import { Module, Logger } from "@nestjs/common";
 import { TypeOrmModule } from "@nestjs/typeorm";
-import { ConfigModule } from "@nestjs/config";
+import { ConfigModule, ConfigService } from "@nestjs/config";
 import { EventEmitterModule } from "@nestjs/event-emitter";
-import { HttpModule } from "@nestjs/axios";
+import { HttpModule, HttpService } from "@nestjs/axios";
 import { PrometheusModule } from "@willsoto/nestjs-prometheus";
 import config from "./config";
 import { HealthModule } from "./health/health.module";
@@ -18,7 +18,8 @@ import { BalanceService, BalancesCleanerService } from "./balance";
 import { TransferService } from "./transfer/transfer.service";
 import { TokenService } from "./token/token.service";
 import { TokenOffChainDataProvider } from "./token/tokenOffChainData/tokenOffChainDataProvider.abstract";
-import { PortalsFiTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider";
+import { CoingeckoTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/coingecko/coingeckoTokenOffChainDataProvider";
+import { PortalsFiTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/portalsFi/portalsFiTokenOffChainDataProvider";
 import { TokenOffChainDataSaverService } from "./token/tokenOffChainData/tokenOffChainDataSaver.service";
 import { CounterModule } from "./counter/counter.module";
 import {
@@ -100,7 +101,16 @@ import { UnitOfWorkModule } from "./unitOfWork";
     TokenService,
     {
       provide: TokenOffChainDataProvider,
-      useClass: PortalsFiTokenOffChainDataProvider,
+      useFactory: (configService: ConfigService, httpService: HttpService) => {
+        const selectedProvider = configService.get<string>("tokens.selectedTokenOffChainDataProvider");
+        switch (selectedProvider) {
+          case "portalsFi":
+            return new PortalsFiTokenOffChainDataProvider(httpService);
+          default:
+            return new CoingeckoTokenOffChainDataProvider(configService, httpService);
+        }
+      },
+      inject: [ConfigService, HttpService],
     },
     TokenOffChainDataSaverService,
     BatchRepository,

diff --git a/packages/worker/src/config.spec.ts b/packages/worker/src/config.spec.ts
@@ -45,7 +45,11 @@ describe("config", () => {
       tokens: {
         enableTokenOffChainDataSaver: false,
         updateTokenOffChainDataInterval: 86_400_000,
-        tokenOffChainDataMinLiquidityFilter: 0,
+        tokenOffChainDataProviders: ["coingecko", "portalsFi"],
+        selectedTokenOffChainDataProvider: "coingecko",
+        coingecko: {
+          isProPlan: false,
+        },
       },
       metrics: {
         collectDbConnectionPoolMetricsInterval: 10000,

diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts
@@ -21,9 +21,11 @@ export default () => {
     DISABLE_BLOCKS_REVERT,
     ENABLE_TOKEN_OFFCHAIN_DATA_SAVER,
     UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL,
-    TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER,
+    SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER,
     FROM_BLOCK,
     TO_BLOCK,
+    COINGECKO_IS_PRO_PLAN,
+    COINGECKO_API_KEY,
   } = process.env;
 
   return {
@@ -59,7 +61,12 @@ export default () => {
     tokens: {
       enableTokenOffChainDataSaver: ENABLE_TOKEN_OFFCHAIN_DATA_SAVER === "true",
       updateTokenOffChainDataInterval: parseInt(UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL, 10) || 86_400_000,
-      tokenOffChainDataMinLiquidityFilter: parseInt(TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER, 10) || 0,
+      tokenOffChainDataProviders: ["coingecko", "portalsFi"],
+      selectedTokenOffChainDataProvider: SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER || "coingecko",
+      coingecko: {
+        isProPlan: COINGECKO_IS_PRO_PLAN === "true",
+        apiKey: COINGECKO_API_KEY,
+      },
     },
     metrics: {
       collectDbConnectionPoolMetricsInterval: parseInt(COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL, 10) || 10000,

diff --git a/packages/worker/src/repositories/token.repository.spec.ts b/packages/worker/src/repositories/token.repository.spec.ts
@@ -174,6 +174,61 @@ describe("TokenRepository", () => {
   });
 
   describe("updateTokenOffChainData", () => {
+    it("throws error when no l1Address or l2Address provided", async () => {
+      const updatedAt = new Date();
+      await expect(
+        repository.updateTokenOffChainData({
+          liquidity: 1000000,
+          usdPrice: 55.89037747,
+          updatedAt,
+        })
+      ).rejects.toThrowError("l1Address or l2Address must be provided");
+    });
+
+    it("updates token offchain data using l1Address when provided", async () => {
+      const updatedAt = new Date();
+      await repository.updateTokenOffChainData({
+        l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1",
+        liquidity: 1000000,
+        usdPrice: 55.89037747,
+        updatedAt,
+      });
+
+      expect(entityManagerMock.update).toBeCalledWith(
+        Token,
+        {
+          l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1",
+        },
+        {
+          liquidity: 1000000,
+          usdPrice: 55.89037747,
+          offChainDataUpdatedAt: updatedAt,
+        }
+      );
+    });
+
+    it("updates token offchain data using l2Address when provided", async () => {
+      const updatedAt = new Date();
+      await repository.updateTokenOffChainData({
+        l2Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1",
+        liquidity: 1000000,
+        usdPrice: 55.89037747,
+        updatedAt,
+      });
+
+      expect(entityManagerMock.update).toBeCalledWith(
+        Token,
+        {
+          l2Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1",
+        },
+        {
+          liquidity: 1000000,
+          usdPrice: 55.89037747,
+          offChainDataUpdatedAt: updatedAt,
+        }
+      );
+    });
+
     it("updates token offchain data when iconURL is not provided", async () => {
       const updatedAt = new Date();
       await repository.updateTokenOffChainData({

diff --git a/packages/worker/src/repositories/token.repository.ts b/packages/worker/src/repositories/token.repository.ts
@@ -63,22 +63,27 @@ export class TokenRepository extends BaseRepository<Token> {
 
   public async updateTokenOffChainData({
     l1Address,
+    l2Address,
     liquidity,
     usdPrice,
     updatedAt,
     iconURL,
   }: {
-    l1Address: string;
-    liquidity: number;
-    usdPrice: number;
-    updatedAt: Date;
+    l1Address?: string;
+    l2Address?: string;
+    liquidity?: number;
+    usdPrice?: number;
+    updatedAt?: Date;
     iconURL?: string;
   }): Promise<void> {
+    if (!l1Address && !l2Address) {
+      throw new Error("l1Address or l2Address must be provided");
+    }
     const transactionManager = this.unitOfWork.getTransactionManager();
     await transactionManager.update(
       this.entityTarget,
       {
-        l1Address,
+        ...(l1Address ? { l1Address } : { l2Address }),
       },
       {
         liquidity,