From 50bb9f790e6b5d1014947342d518035bc757ea6b Mon Sep 17 00:00:00 2001 From: nigiri <168690269+0xnigir1@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:23:56 -0300 Subject: [PATCH] feat: price in memory caching (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # 🤖 Linear Closes ZKS-120 ## Description Add an InMemory caching layer for token prices: - Uses NestJS Caching Module - TTL of 60secs --------- Co-authored-by: 0xyaco --- .../src/interfaces/pricing.interface.ts | 3 - libs/pricing/src/pricing.module.ts | 10 ++- .../src/services/coingecko.service.spec.ts | 57 +++++++++------ .../pricing/src/services/coingecko.service.ts | 73 ++++++++++++++++--- libs/shared/src/constants.ts | 0 libs/shared/src/constants/index.ts | 2 + package.json | 2 + pnpm-lock.yaml | 43 +++++++++++ 8 files changed, 153 insertions(+), 37 deletions(-) delete mode 100644 libs/shared/src/constants.ts create mode 100644 libs/shared/src/constants/index.ts diff --git a/libs/pricing/src/interfaces/pricing.interface.ts b/libs/pricing/src/interfaces/pricing.interface.ts index 34055d2..052fc32 100644 --- a/libs/pricing/src/interfaces/pricing.interface.ts +++ b/libs/pricing/src/interfaces/pricing.interface.ts @@ -8,12 +8,9 @@ export interface IPricingService { /** * Retrieves the prices of the specified tokens. * @param tokenIds - An array of token IDs. - * @param [config] - Optional configuration object. - * @param config.currency - The currency in which the prices should be returned. * @returns A promise that resolves to a record containing the token IDs as keys and their corresponding prices as values. */ getTokenPrices( tokenIds: TokenId[], - config?: { currency: string }, ): Promise>; } diff --git a/libs/pricing/src/pricing.module.ts b/libs/pricing/src/pricing.module.ts index 6a5b670..7e1fd53 100644 --- a/libs/pricing/src/pricing.module.ts +++ b/libs/pricing/src/pricing.module.ts @@ -1,11 +1,19 @@ +import { CacheModule } from "@nestjs/cache-manager"; import { Module } from "@nestjs/common"; import { LoggerModule } from "@zkchainhub/shared"; +import { TOKEN_CACHE_TTL_IN_SEC } from "@zkchainhub/shared/constants/"; import { CoingeckoService } from "./services"; @Module({ - imports: [LoggerModule], + imports: [ + LoggerModule, + CacheModule.register({ + store: "memory", + ttl: TOKEN_CACHE_TTL_IN_SEC, + }), + ], providers: [CoingeckoService], exports: [CoingeckoService], }) diff --git a/libs/pricing/src/services/coingecko.service.spec.ts b/libs/pricing/src/services/coingecko.service.spec.ts index 55da593..3321b95 100644 --- a/libs/pricing/src/services/coingecko.service.spec.ts +++ b/libs/pricing/src/services/coingecko.service.spec.ts @@ -1,13 +1,16 @@ +import { createMock } from "@golevelup/ts-jest"; +import { Cache, CACHE_MANAGER } from "@nestjs/cache-manager"; import { Logger } from "@nestjs/common"; import { Test, TestingModule } from "@nestjs/testing"; -import { AxiosError, AxiosInstance } from "axios"; +import { AxiosInstance } from "axios"; import MockAdapter from "axios-mock-adapter"; import { WINSTON_MODULE_PROVIDER } from "nest-winston"; import { ApiNotAvailable, RateLimitExceeded } from "@zkchainhub/pricing/exceptions"; import { TokenPrices } from "@zkchainhub/pricing/types/tokenPrice.type"; +import { BASE_CURRENCY } from "@zkchainhub/shared"; -import { CoingeckoService } from "./coingecko.service"; +import { CoingeckoService, DECIMALS_PRECISION } from "./coingecko.service"; export const mockLogger: Partial = { log: jest.fn(), @@ -20,6 +23,7 @@ describe("CoingeckoService", () => { let service: CoingeckoService; let axios: AxiosInstance; let mockAxios: MockAdapter; + let cache: Cache; const apiKey = "COINGECKO_API_KEY"; const apiBaseUrl = "https://api.coingecko.com/api/v3/"; @@ -29,21 +33,31 @@ describe("CoingeckoService", () => { CoingeckoService, { provide: CoingeckoService, - useFactory: (logger: Logger) => { - return new CoingeckoService(apiKey, apiBaseUrl, logger); + useFactory: (logger: Logger, cache: Cache) => { + return new CoingeckoService(apiKey, apiBaseUrl, logger, cache); }, - inject: [WINSTON_MODULE_PROVIDER], + inject: [WINSTON_MODULE_PROVIDER, CACHE_MANAGER], }, { provide: WINSTON_MODULE_PROVIDER, useValue: mockLogger, }, + { + provide: CACHE_MANAGER, + useValue: createMock({ + store: createMock({ + mget: jest.fn(), + mset: jest.fn(), + }), + }), + }, ], }).compile(); service = module.get(CoingeckoService); axios = service["axios"]; mockAxios = new MockAdapter(axios); + cache = module.get(CACHE_MANAGER); }); afterEach(() => { @@ -67,19 +81,19 @@ describe("CoingeckoService", () => { }); describe("getTokenPrices", () => { - it("return token prices", async () => { + it("fetches all token prices from Coingecko", async () => { const tokenIds = ["token1", "token2"]; - const currency = "usd"; const expectedResponse: TokenPrices = { token1: { usd: 1.23 }, token2: { usd: 4.56 }, }; + jest.spyOn(cache.store, "mget").mockResolvedValueOnce([null, null]); jest.spyOn(axios, "get").mockResolvedValueOnce({ data: expectedResponse, }); - const result = await service.getTokenPrices(tokenIds, { currency }); + const result = await service.getTokenPrices(tokenIds); expect(result).toEqual({ token1: 1.23, @@ -87,51 +101,50 @@ describe("CoingeckoService", () => { }); expect(axios.get).toHaveBeenCalledWith(`simple/price`, { params: { - vs_currencies: currency, + vs_currencies: BASE_CURRENCY, ids: tokenIds.join(","), - precision: service["DECIMALS_PRECISION"].toString(), + precision: DECIMALS_PRECISION.toString(), }, }); + expect(cache.store.mget).toHaveBeenCalledWith("token1", "token2"); + expect(cache.store.mset).toHaveBeenCalledWith([ + ["token1", 1.23], + ["token2", 4.56], + ]); }); it("throw ApiNotAvailable when Coingecko returns a 500 family exception", async () => { const tokenIds = ["token1", "token2"]; - const currency = "usd"; + jest.spyOn(cache.store, "mget").mockResolvedValueOnce([null, null]); mockAxios.onGet().replyOnce(503, { data: {}, status: 503, statusText: "Service not available", }); - await expect(service.getTokenPrices(tokenIds, { currency })).rejects.toThrow( + await expect(service.getTokenPrices(tokenIds)).rejects.toThrow( new ApiNotAvailable("Coingecko"), ); }); it("throw RateLimitExceeded when Coingecko returns 429 exception", async () => { const tokenIds = ["token1", "token2"]; - const currency = "usd"; + jest.spyOn(cache.store, "mget").mockResolvedValueOnce([null, null]); mockAxios.onGet().replyOnce(429, { data: {}, status: 429, statusText: "Too Many Requests", }); - await expect(service.getTokenPrices(tokenIds, { currency })).rejects.toThrow( - new RateLimitExceeded(), - ); + await expect(service.getTokenPrices(tokenIds)).rejects.toThrow(new RateLimitExceeded()); }); it("throw an HttpException with the error message when an error occurs", async () => { const tokenIds = ["invalidTokenId", "token2"]; - const currency = "usd"; - - jest.spyOn(axios, "get").mockRejectedValueOnce( - new AxiosError("Invalid token ID", "400"), - ); + jest.spyOn(cache.store, "mget").mockResolvedValueOnce([null, null]); mockAxios.onGet().replyOnce(400, { data: { message: "Invalid token ID", @@ -140,7 +153,7 @@ describe("CoingeckoService", () => { statusText: "Bad Request", }); - await expect(service.getTokenPrices(tokenIds, { currency })).rejects.toThrow(); + await expect(service.getTokenPrices(tokenIds)).rejects.toThrow(); }); }); }); diff --git a/libs/pricing/src/services/coingecko.service.ts b/libs/pricing/src/services/coingecko.service.ts index 567b5bf..f1b64c6 100644 --- a/libs/pricing/src/services/coingecko.service.ts +++ b/libs/pricing/src/services/coingecko.service.ts @@ -1,3 +1,4 @@ +import { Cache, CACHE_MANAGER } from "@nestjs/cache-manager"; import { Inject, Injectable, LoggerService } from "@nestjs/common"; import axios, { AxiosInstance, isAxiosError } from "axios"; import { WINSTON_MODULE_NEST_PROVIDER } from "nest-winston"; @@ -5,14 +6,17 @@ import { WINSTON_MODULE_NEST_PROVIDER } from "nest-winston"; import { ApiNotAvailable, RateLimitExceeded } from "@zkchainhub/pricing/exceptions"; import { IPricingService } from "@zkchainhub/pricing/interfaces"; import { TokenPrices } from "@zkchainhub/pricing/types/tokenPrice.type"; +import { BASE_CURRENCY } from "@zkchainhub/shared"; + +export const AUTH_HEADER = "x-cg-pro-api-key"; +export const DECIMALS_PRECISION = 3; /** * Service for fetching token prices from Coingecko API. + * Prices are always denominated in USD. */ @Injectable() export class CoingeckoService implements IPricingService { - private readonly AUTH_HEADER = "x-cg-pro-api-key"; - private readonly DECIMALS_PRECISION = 3; private readonly axios: AxiosInstance; /** @@ -24,12 +28,13 @@ export class CoingeckoService implements IPricingService { private readonly apiKey: string, private readonly apiBaseUrl: string = "https://api.coingecko.com/api/v3/", @Inject(WINSTON_MODULE_NEST_PROVIDER) private readonly logger: LoggerService, + @Inject(CACHE_MANAGER) private readonly cacheManager: Cache, ) { this.axios = axios.create({ baseURL: apiBaseUrl, headers: { common: { - [this.AUTH_HEADER]: apiKey, + [AUTH_HEADER]: apiKey, Accept: "application/json", }, }, @@ -42,20 +47,66 @@ export class CoingeckoService implements IPricingService { /** * @param tokenIds - An array of Coingecko Tokens IDs. - * @param config.currency - The currency in which the prices should be returned (default: "usd"). + * @returns A promise that resolves to a record of token prices in USD. */ - async getTokenPrices( - tokenIds: string[], - config: { currency: string } = { currency: "usd" }, - ): Promise> { - const { currency } = config; + async getTokenPrices(tokenIds: string[]): Promise> { + const cachedTokenPrices = await this.getTokenPricesFromCache(tokenIds); + const missingTokenIds: string[] = []; + const cachedMap = cachedTokenPrices.reduce( + (result, price, index) => { + if (price !== null) result[tokenIds.at(index) as string] = price; + else missingTokenIds.push(tokenIds.at(index) as string); + + return result; + }, + {} as Record, + ); + + const missingTokenPrices = await this.fetchTokenPrices(missingTokenIds); + + await this.saveTokenPricesToCache(missingTokenPrices); + + return { ...cachedMap, ...missingTokenPrices }; + } + + private formatTokenCacheKey(tokenId: string, currency: string) { + return `${tokenId}.${currency}`; + } + + /** + * Retrieves multiple token prices from the cache at once. + * @param keys - An array of cache keys. + * @returns A promise that resolves to an array of token prices (number or null). + */ + private async getTokenPricesFromCache(keys: string[]): Promise<(number | null)[]> { + return this.cacheManager.store.mget(...keys) as Promise<(number | null)[]>; + } + + /** + * Saves multiple token prices to the cache at once. + * + * @param prices - The token prices to be saved. + * @param currency - The currency in which the prices are denominated. + */ + private async saveTokenPricesToCache(prices: Record) { + if (Object.keys(prices).length === 0) return; + + this.cacheManager.store.mset( + Object.entries(prices).map(([tokenId, price]) => [tokenId, price]), + ); + } + + private async fetchTokenPrices(tokenIds: string[]): Promise> { + if (tokenIds.length === 0) { + return {}; + } return this.axios .get("simple/price", { params: { - vs_currencies: currency, + vs_currencies: BASE_CURRENCY, ids: tokenIds.join(","), - precision: this.DECIMALS_PRECISION.toString(), + precision: DECIMALS_PRECISION.toString(), }, }) .then((response) => { diff --git a/libs/shared/src/constants.ts b/libs/shared/src/constants.ts deleted file mode 100644 index e69de29..0000000 diff --git a/libs/shared/src/constants/index.ts b/libs/shared/src/constants/index.ts new file mode 100644 index 0000000..bdeb317 --- /dev/null +++ b/libs/shared/src/constants/index.ts @@ -0,0 +1,2 @@ +export const TOKEN_CACHE_TTL_IN_SEC = 60; +export const BASE_CURRENCY = "usd"; diff --git a/package.json b/package.json index c72a7c8..ad48529 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@nestjs/axios": "3.0.2", + "@nestjs/cache-manager": "2.2.2", "@nestjs/common": "10.0.0", "@nestjs/core": "10.0.0", "@nestjs/platform-express": "10.0.0", @@ -32,6 +33,7 @@ "axios": "1.7.2", "axios-mock-adapter": "1.22.0", "nest-winston": "1.9.7", + "cache-manager": "5.7.4", "reflect-metadata": "0.1.13", "rxjs": "7.8.1", "viem": "2.17.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 56ceb91..447a081 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@nestjs/axios': specifier: 3.0.2 version: 3.0.2(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(axios@1.7.2)(rxjs@7.8.1) + '@nestjs/cache-manager': + specifier: 2.2.2 + version: 2.2.2(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(@nestjs/core@10.0.0(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(@nestjs/platform-express@10.0.0)(reflect-metadata@0.1.13)(rxjs@7.8.1))(cache-manager@5.7.4)(rxjs@7.8.1) '@nestjs/common': specifier: 10.0.0 version: 10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1) @@ -32,6 +35,9 @@ importers: axios-mock-adapter: specifier: 1.22.0 version: 1.22.0(axios@1.7.2) + cache-manager: + specifier: 5.7.4 + version: 5.7.4 nest-winston: specifier: 1.9.7 version: 1.9.7(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(winston@3.13.1) @@ -566,6 +572,14 @@ packages: axios: ^1.3.1 rxjs: ^6.0.0 || ^7.0.0 + '@nestjs/cache-manager@2.2.2': + resolution: {integrity: sha512-+n7rpU1QABeW2WV17Dl1vZCG3vWjJU1MaamWgZvbGxYE9EeCM0lVLfw3z7acgDTNwOy+K68xuQPoIMxD0bhjlA==} + peerDependencies: + '@nestjs/common': ^9.0.0 || ^10.0.0 + '@nestjs/core': ^9.0.0 || ^10.0.0 + cache-manager: <=5 + rxjs: ^7.0.0 + '@nestjs/cli@10.0.0': resolution: {integrity: sha512-14pju3ejAAUpFe1iK99v/b7Bw96phBMV58GXTSm3TcdgaI4O7UTLXTbMiUNyU+LGr/1CPIfThcWqFyKhDIC9VQ==} engines: {node: '>= 16'} @@ -1234,6 +1248,10 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} + cache-manager@5.7.4: + resolution: {integrity: sha512-7B29xK1D8hOVdrP0SAy2DGJ/QZxy2TqxS8s2drlLGYI/xOTSJmXfatks7aKKNHvXN6SnKnPtYCi0T82lslB3Fw==} + engines: {node: '>= 18'} + call-bind@1.0.7: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} engines: {node: '>= 0.4'} @@ -2377,6 +2395,9 @@ packages: lodash.camelcase@4.3.0: resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.clonedeep@4.5.0: + resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} + lodash.isplainobject@4.0.6: resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} @@ -2754,6 +2775,10 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + promise-coalesce@1.1.2: + resolution: {integrity: sha512-zLaJ9b8hnC564fnJH6NFSOGZYYdzrAJn2JUUIwzoQb32fG2QAakpDNM+CZo1km6keXkRXRM+hml1BFAPVnPkxg==} + engines: {node: '>=16'} + prompts@2.4.2: resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} engines: {node: '>= 6'} @@ -4065,6 +4090,13 @@ snapshots: axios: 1.7.2 rxjs: 7.8.1 + '@nestjs/cache-manager@2.2.2(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(@nestjs/core@10.0.0(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(@nestjs/platform-express@10.0.0)(reflect-metadata@0.1.13)(rxjs@7.8.1))(cache-manager@5.7.4)(rxjs@7.8.1)': + dependencies: + '@nestjs/common': 10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1) + '@nestjs/core': 10.0.0(@nestjs/common@10.0.0(reflect-metadata@0.1.13)(rxjs@7.8.1))(@nestjs/platform-express@10.0.0)(reflect-metadata@0.1.13)(rxjs@7.8.1) + cache-manager: 5.7.4 + rxjs: 7.8.1 + '@nestjs/cli@10.0.0(@swc/core@1.6.13)': dependencies: '@angular-devkit/core': 16.1.0(chokidar@3.5.3) @@ -4834,6 +4866,13 @@ snapshots: bytes@3.1.2: {} + cache-manager@5.7.4: + dependencies: + eventemitter3: 5.0.1 + lodash.clonedeep: 4.5.0 + lru-cache: 10.4.0 + promise-coalesce: 1.1.2 + call-bind@1.0.7: dependencies: es-define-property: 1.0.0 @@ -6205,6 +6244,8 @@ snapshots: lodash.camelcase@4.3.0: {} + lodash.clonedeep@4.5.0: {} + lodash.isplainobject@4.0.6: {} lodash.kebabcase@4.1.1: {} @@ -6524,6 +6565,8 @@ snapshots: process-nextick-args@2.0.1: {} + promise-coalesce@1.1.2: {} + prompts@2.4.2: dependencies: kleur: 3.0.3