diff --git a/package-lock.json b/package-lock.json index 47fdb2ecf2..4740048ce2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55033,6 +55033,7 @@ "version": "0.0.0", "license": "MIT", "dependencies": { + "@nestjs/axios": "^3.0.0", "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.0.0", @@ -55041,6 +55042,7 @@ "@nestjs/terminus": "^9.1.2", "@nestjs/typeorm": "^9.0.1", "@willsoto/nestjs-prometheus": "^4.7.0", + "axios": "^1.4.0", "ethers": "^5.7.1", "nest-winston": "^1.7.0", "pg": "^8.8.0", diff --git a/packages/api/src/api/api.controller.ts b/packages/api/src/api/api.controller.ts index 09891c6463..b65174a280 100644 --- a/packages/api/src/api/api.controller.ts +++ b/packages/api/src/api/api.controller.ts @@ -234,7 +234,9 @@ export class ApiController { @ApiTags("Account API") @Get("api?module=account&action=txlistinternal") - @ApiOperation({ summary: "Retrieve internal transactions" }) + @ApiOperation({ + summary: "Retrieve internal transactions for a given blocks range (only transfers are supported for now)", + }) @ApiQuery({ name: "startblock", type: "integer", @@ -265,7 +267,9 @@ export class ApiController { @ApiTags("Account API") @Get("api?module=account&action=txlistinternal&address=") - @ApiOperation({ summary: "Retrieve internal transactions for a given address" }) + @ApiOperation({ + summary: "Retrieve internal transactions for a given address (only transfers are supported for now)", + }) @ApiQuery({ name: "address", description: "The address to filter internal transactions by", @@ -302,7 +306,9 @@ export class ApiController { @ApiTags("Account API") @Get("api?module=account&action=txlistinternal&txhash=") - @ApiOperation({ summary: "Retrieve internal transactions for a given transaction hash" }) + @ApiOperation({ + summary: "Retrieve internal transactions for a given transaction hash (only transfers are supported for now)", + }) @ApiQuery({ name: "txhash", description: "The transaction hash to filter internal transaction by", diff --git a/packages/api/src/api/mappers/transferMapper.spec.ts b/packages/api/src/api/mappers/transferMapper.spec.ts index 7a7967e035..1d90965e61 100644 --- a/packages/api/src/api/mappers/transferMapper.spec.ts +++ b/packages/api/src/api/mappers/transferMapper.spec.ts @@ -163,5 +163,19 @@ describe("transferMapper", () => { }); }); }); + + describe("when transfer amount is NULL", () => { + it("sets value as undefined", () => { + expect( + mapTransferListItem( + { + ...transfer, + amount: null, + } as unknown as Transfer, + 100 + ).value + ).toBe(undefined); + }); + }); }); }); diff --git a/packages/api/src/api/mappers/transferMapper.ts b/packages/api/src/api/mappers/transferMapper.ts index 13551cf224..cb4a418bb9 100644 --- a/packages/api/src/api/mappers/transferMapper.ts +++ b/packages/api/src/api/mappers/transferMapper.ts @@ -10,7 +10,7 @@ export const mapTransferListItem = (transfer: Transfer, lastBlockNumber: number) transactionIndex: transfer.transaction?.transactionIndex.toString(), from: transfer.from, to: transfer.to, - value: transfer.amount, + value: transfer.amount || undefined, tokenID: transfer.fields?.tokenId, tokenName: transfer.token?.name, tokenSymbol: transfer.token?.symbol, diff --git a/packages/api/src/token/token.entity.ts b/packages/api/src/token/token.entity.ts index d2280eb047..874cc55b35 100644 --- a/packages/api/src/token/token.entity.ts +++ b/packages/api/src/token/token.entity.ts @@ -14,10 +14,13 @@ export const ETH_TOKEN: Token = { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, } as Token; @Entity({ name: "tokens" }) -@Index(["blockNumber", "logIndex"]) +@Index(["liquidity", "blockNumber", "logIndex"]) export class Token extends BaseEntity { @PrimaryColumn({ type: "bytea", transformer: normalizeAddressTransformer }) public readonly l2Address: string; @@ -25,6 +28,7 @@ export class Token extends BaseEntity { @Column({ generated: true, type: "bigint", select: false }) public readonly number: number; + @Index() @Column({ type: "bytea", nullable: true, transformer: normalizeAddressTransformer }) public readonly l1Address?: string; @@ -42,4 +46,17 @@ export class Token extends BaseEntity { @Column({ type: "int", select: false }) public readonly logIndex: number; + + @Column({ type: "double precision", nullable: true }) + public readonly usdPrice?: number; + + @Column({ type: "double precision", nullable: true }) + public readonly liquidity?: number; + + @Column({ nullable: true }) + public readonly iconURL?: string; + + @Index() + @Column({ type: "timestamp", nullable: true, select: false }) + public readonly offChainDataUpdatedAt?: Date; } diff --git a/packages/api/src/token/token.service.spec.ts b/packages/api/src/token/token.service.spec.ts index d756d85f8d..247ea757fc 100644 --- a/packages/api/src/token/token.service.spec.ts +++ b/packages/api/src/token/token.service.spec.ts @@ -68,6 +68,9 @@ describe("TokenService", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }); }); @@ -139,11 +142,12 @@ describe("TokenService", () => { expect(repositoryMock.createQueryBuilder).toHaveBeenCalledWith("token"); }); - it("returns tokens ordered by blockNumber and logIndex DESC", async () => { + it("returns tokens ordered by liquidity, blockNumber and logIndex DESC", async () => { await service.findAll(pagingOptions); expect(queryBuilderMock.orderBy).toBeCalledTimes(1); - expect(queryBuilderMock.orderBy).toHaveBeenCalledWith("token.blockNumber", "DESC"); - expect(queryBuilderMock.addOrderBy).toBeCalledTimes(1); + expect(queryBuilderMock.orderBy).toHaveBeenCalledWith("token.liquidity", "DESC"); + expect(queryBuilderMock.addOrderBy).toBeCalledTimes(2); + expect(queryBuilderMock.addOrderBy).toHaveBeenCalledWith("token.blockNumber", "DESC"); expect(queryBuilderMock.addOrderBy).toHaveBeenCalledWith("token.logIndex", "DESC"); }); diff --git a/packages/api/src/token/token.service.ts b/packages/api/src/token/token.service.ts index b4d2d89288..8d6ad3628e 100644 --- a/packages/api/src/token/token.service.ts +++ b/packages/api/src/token/token.service.ts @@ -31,7 +31,8 @@ export class TokenService { public async findAll(paginationOptions: IPaginationOptions): Promise> { const queryBuilder = this.tokenRepository.createQueryBuilder("token"); - queryBuilder.orderBy("token.blockNumber", "DESC"); + queryBuilder.orderBy("token.liquidity", "DESC"); + queryBuilder.addOrderBy("token.blockNumber", "DESC"); queryBuilder.addOrderBy("token.logIndex", "DESC"); return await paginate(queryBuilder, paginationOptions); } diff --git a/packages/api/test/address.e2e-spec.ts b/packages/api/test/address.e2e-spec.ts index 32f32a33a8..1e4193f5da 100644 --- a/packages/api/test/address.e2e-spec.ts +++ b/packages/api/test/address.e2e-spec.ts @@ -391,6 +391,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956": { @@ -401,6 +404,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956", name: "TEST 6", symbol: "TEST 6", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954": { @@ -411,6 +417,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954", name: "TEST 4", symbol: "TEST 4", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955": { @@ -421,6 +430,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955", name: "TEST 5", symbol: "TEST 5", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -448,6 +460,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956": { @@ -458,6 +473,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956", name: "TEST 6", symbol: "TEST 6", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954": { @@ -468,6 +486,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954", name: "TEST 4", symbol: "TEST 4", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955": { @@ -478,6 +499,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955", name: "TEST 5", symbol: "TEST 5", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -505,6 +529,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956": { @@ -515,6 +542,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956", name: "TEST 6", symbol: "TEST 6", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954": { @@ -525,6 +555,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954", name: "TEST 4", symbol: "TEST 4", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955": { @@ -535,6 +568,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955", name: "TEST 5", symbol: "TEST 5", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -562,6 +598,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956": { @@ -572,6 +611,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956", name: "TEST 6", symbol: "TEST 6", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954": { @@ -582,6 +624,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954", name: "TEST 4", symbol: "TEST 4", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955": { @@ -592,6 +637,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955", name: "TEST 5", symbol: "TEST 5", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -621,6 +669,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956": { @@ -631,6 +682,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488FC54FcCc6f319D4863Ddc2c2899Ed35d8956", name: "TEST 6", symbol: "TEST 6", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954": { @@ -641,6 +695,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488Fc54FCcC6f319d4863dDc2C2899ED35D8954", name: "TEST 4", symbol: "TEST 4", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955": { @@ -651,6 +708,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54fCcC6F319D4863dDC2c2899ED35D8955", name: "TEST 5", symbol: "TEST 5", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -680,6 +740,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fcCC6f319D4863Ddc2C2899ED35d8957", name: "TEST 7", symbol: "TEST 7", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959": { @@ -690,6 +753,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959", name: "TEST 9", symbol: "TEST 9", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958": { @@ -700,6 +766,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958", name: "TEST 8", symbol: "TEST 8", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -730,6 +799,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fcCC6f319D4863Ddc2C2899ED35d8957", name: "TEST 7", symbol: "TEST 7", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959": { @@ -740,6 +812,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959", name: "TEST 9", symbol: "TEST 9", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958": { @@ -750,6 +825,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958", name: "TEST 8", symbol: "TEST 8", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -780,6 +858,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fcCC6f319D4863Ddc2C2899ED35d8957", name: "TEST 7", symbol: "TEST 7", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959": { @@ -790,6 +871,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959", name: "TEST 9", symbol: "TEST 9", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958": { @@ -800,6 +884,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958", name: "TEST 8", symbol: "TEST 8", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -830,6 +917,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fcCC6f319D4863Ddc2C2899ED35d8957", name: "TEST 7", symbol: "TEST 7", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959": { @@ -840,6 +930,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fC54fccC6F319d4863DDc2C2899ed35d8959", name: "TEST 9", symbol: "TEST 9", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958": { @@ -850,6 +943,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x9488fc54FCCC6f319D4863dDc2c2899Ed35D8958", name: "TEST 8", symbol: "TEST 8", + iconURL: null, + liquidity: null, + usdPrice: null, }, }, }, @@ -1082,6 +1178,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x97d0a23F34E535e44dF8ba84c53A0945cF0eEb67", name: "TEST", symbol: "TST", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x97d0a23F34E535e44dF8ba84c53A0945cF0eEb67", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e11", @@ -1102,6 +1201,9 @@ describe("AddressController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e11", @@ -1122,6 +1224,9 @@ describe("AddressController (e2e)", () => { l2Address: "0x97d0a23F34E535e44dF8ba84c53A0945cF0eEb67", name: "TEST", symbol: "TST", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x97d0a23F34E535e44dF8ba84c53A0945cF0eEb67", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e11", diff --git a/packages/api/test/token.e2e-spec.ts b/packages/api/test/token.e2e-spec.ts index 5d2167436a..35d9e671c5 100644 --- a/packages/api/test/token.e2e-spec.ts +++ b/packages/api/test/token.e2e-spec.ts @@ -254,6 +254,9 @@ describe("TokenController (e2e)", () => { symbol: "TEST1", name: "TEST token 1", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }) ); }); @@ -269,6 +272,9 @@ describe("TokenController (e2e)", () => { symbol: "TEST1", name: "TEST token 1", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }) ); }); @@ -284,6 +290,9 @@ describe("TokenController (e2e)", () => { symbol: "TEST1", name: "TEST token 1", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }) ); }); @@ -299,6 +308,9 @@ describe("TokenController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }) ); }); @@ -341,6 +353,9 @@ describe("TokenController (e2e)", () => { name: "TEST token 28", symbol: "TEST28", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, { l1Address: "0xF754ff5E8a6F257E162f72578A4bB0493C068127", @@ -348,6 +363,9 @@ describe("TokenController (e2e)", () => { name: "TEST token 27", symbol: "TEST27", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, ]) ); @@ -427,6 +445,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -447,6 +468,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -469,6 +493,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -489,6 +516,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -509,6 +539,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -529,6 +562,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -549,6 +585,9 @@ describe("TokenController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token 1", symbol: "TEST1", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -610,6 +649,9 @@ describe("TokenController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", tokenType: "ETH", @@ -630,6 +672,9 @@ describe("TokenController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", tokenType: "ETH", @@ -650,6 +695,9 @@ describe("TokenController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", tokenType: "ETH", diff --git a/packages/api/test/transaction.e2e-spec.ts b/packages/api/test/transaction.e2e-spec.ts index d3d978ec27..1d6e0f978d 100644 --- a/packages/api/test/transaction.e2e-spec.ts +++ b/packages/api/test/transaction.e2e-spec.ts @@ -1016,6 +1016,9 @@ describe("TransactionController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1036,6 +1039,9 @@ describe("TransactionController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token", symbol: "TEST", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1056,6 +1062,9 @@ describe("TransactionController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1076,6 +1085,9 @@ describe("TransactionController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token", symbol: "TEST", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1096,6 +1108,9 @@ describe("TransactionController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1116,6 +1131,9 @@ describe("TransactionController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token", symbol: "TEST", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1136,6 +1154,9 @@ describe("TransactionController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1156,6 +1177,9 @@ describe("TransactionController (e2e)", () => { l2Address: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", name: "TEST token", symbol: "TEST", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1176,6 +1200,9 @@ describe("TransactionController (e2e)", () => { symbol: "ETH", name: "Ether", decimals: 18, + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", @@ -1241,6 +1268,9 @@ describe("TransactionController (e2e)", () => { l2Address: "0x000000000000000000000000000000000000800A", name: "Ether", symbol: "ETH", + iconURL: null, + liquidity: null, + usdPrice: null, }, tokenAddress: "0x000000000000000000000000000000000000800A", transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10", diff --git a/packages/worker/.env.example b/packages/worker/.env.example index 5d6bf86cd3..6cb58083b4 100644 --- a/packages/worker/.env.example +++ b/packages/worker/.env.example @@ -30,5 +30,9 @@ DISABLE_BALANCES_PROCESSING=false DISABLE_OLD_BALANCES_CLEANER=false DISABLE_BLOCKS_REVERT=false +ENABLE_TOKEN_OFFCHAIN_DATA_SAVER=false +UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL=86400000 +TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER=1000000 + FROM_BLOCK=0 TO_BLOCK= diff --git a/packages/worker/package.json b/packages/worker/package.json index d24599836a..53f9decf39 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -31,6 +31,7 @@ "migration:revert": "npm run typeorm migration:revert -- -d ./src/typeorm.config.ts" }, "dependencies": { + "@nestjs/axios": "^3.0.0", "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.0.0", @@ -39,6 +40,7 @@ "@nestjs/terminus": "^9.1.2", "@nestjs/typeorm": "^9.0.1", "@willsoto/nestjs-prometheus": "^4.7.0", + "axios": "^1.4.0", "ethers": "^5.7.1", "nest-winston": "^1.7.0", "pg": "^8.8.0", diff --git a/packages/worker/src/app.module.ts b/packages/worker/src/app.module.ts index 434f99ddfa..43ce3900fb 100644 --- a/packages/worker/src/app.module.ts +++ b/packages/worker/src/app.module.ts @@ -2,6 +2,7 @@ import { Module, Logger } from "@nestjs/common"; import { TypeOrmModule } from "@nestjs/typeorm"; import { ConfigModule } from "@nestjs/config"; import { EventEmitterModule } from "@nestjs/event-emitter"; +import { HttpModule } from "@nestjs/axios"; import { PrometheusModule } from "@willsoto/nestjs-prometheus"; import config from "./config"; import { HealthModule } from "./health/health.module"; @@ -16,6 +17,9 @@ import { AddressService } from "./address/address.service"; import { BalanceService, BalancesCleanerService } from "./balance"; import { TransferService } from "./transfer/transfer.service"; import { TokenService } from "./token/token.service"; +import { TokenOffChainDataProvider } from "./token/tokenOffChainData/tokenOffChainDataProvider.abstract"; +import { PortalsFiTokenOffChainDataProvider } from "./token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider"; +import { TokenOffChainDataSaverService } from "./token/tokenOffChainData/tokenOffChainDataSaver.service"; import { CounterModule } from "./counter/counter.module"; import { BatchRepository, @@ -84,6 +88,7 @@ import { UnitOfWorkModule } from "./unitOfWork"; UnitOfWorkModule, CounterModule, HealthModule, + HttpModule, ], providers: [ AppService, @@ -93,6 +98,11 @@ import { UnitOfWorkModule } from "./unitOfWork"; BalancesCleanerService, TransferService, TokenService, + { + provide: TokenOffChainDataProvider, + useClass: PortalsFiTokenOffChainDataProvider, + }, + TokenOffChainDataSaverService, BatchRepository, BlockRepository, TransactionRepository, diff --git a/packages/worker/src/app.service.spec.ts b/packages/worker/src/app.service.spec.ts index fc136c47ad..3d4afb8363 100644 --- a/packages/worker/src/app.service.spec.ts +++ b/packages/worker/src/app.service.spec.ts @@ -10,6 +10,7 @@ import { CounterService } from "./counter"; import { BatchService } from "./batch"; import { BlockService } from "./block"; import { BlocksRevertService } from "./blocksRevert"; +import { TokenOffChainDataSaverService } from "./token/tokenOffChainData/tokenOffChainDataSaver.service"; import runMigrations from "./utils/runMigrations"; import { BLOCKS_REVERT_DETECTED_EVENT } from "./constants"; @@ -35,6 +36,7 @@ describe("AppService", () => { let batchService: BatchService; let blockService: BlockService; let blocksRevertService: BlocksRevertService; + let tokenOffChainDataSaverService: TokenOffChainDataSaverService; let dataSourceMock: DataSource; let configServiceMock: ConfigService; @@ -58,6 +60,10 @@ describe("AppService", () => { blocksRevertService = mock({ handleRevert: jest.fn().mockResolvedValue(null), }); + tokenOffChainDataSaverService = mock({ + start: jest.fn().mockResolvedValue(null), + stop: jest.fn().mockResolvedValue(null), + }); dataSourceMock = mock(); configServiceMock = mock({ get: jest.fn().mockReturnValue(false), @@ -88,6 +94,10 @@ describe("AppService", () => { provide: BlocksRevertService, useValue: blocksRevertService, }, + { + provide: TokenOffChainDataSaverService, + useValue: tokenOffChainDataSaverService, + }, { provide: DataSource, useValue: dataSourceMock, @@ -156,6 +166,13 @@ describe("AppService", () => { expect(balancesCleanerService.stop).toBeCalledTimes(1); }); + it("does not start token offchain data saver service by default", async () => { + appService.onModuleInit(); + await migrationsRunFinished; + expect(tokenOffChainDataSaverService.start).not.toBeCalled(); + appService.onModuleDestroy(); + }); + it("does not start batches service when disableBatchesProcessing is true", async () => { (configServiceMock.get as jest.Mock).mockReturnValue(true); appService.onModuleInit(); @@ -179,6 +196,15 @@ describe("AppService", () => { expect(balancesCleanerService.start).not.toBeCalled(); appService.onModuleDestroy(); }); + + it("starts token offchain data saver service when enableTokenOffChainDataSaver is true", async () => { + (configServiceMock.get as jest.Mock).mockReturnValue(true); + appService.onModuleInit(); + await migrationsRunFinished; + expect(tokenOffChainDataSaverService.start).toBeCalledTimes(1); + appService.onModuleDestroy(); + expect(tokenOffChainDataSaverService.stop).toBeCalledTimes(1); + }); }); describe("onModuleDestroy", () => { @@ -201,17 +227,26 @@ describe("AppService", () => { appService.onModuleDestroy(); expect(balancesCleanerService.stop).toBeCalledTimes(1); }); + + it("stops token offchain data saver service", async () => { + appService.onModuleDestroy(); + expect(tokenOffChainDataSaverService.stop).toBeCalledTimes(1); + }); }); describe("Handling blocks revert event", () => { it("stops all the workers, handles blocks revert and then restarts the workers", async () => { (runMigrations as jest.Mock).mockResolvedValue(null); + (configServiceMock.get as jest.Mock).mockImplementation((key) => + key === "tokens.enableTokenOffChainDataSaver" ? true : false + ); await app.init(); expect(blockService.stop).toBeCalledTimes(1); expect(batchService.stop).toBeCalledTimes(1); expect(counterService.stop).toBeCalledTimes(1); expect(balancesCleanerService.stop).toBeCalledTimes(1); + expect(tokenOffChainDataSaverService.stop).toBeCalledTimes(1); expect(blocksRevertService.handleRevert).toBeCalledWith(blockNumber); @@ -219,6 +254,7 @@ describe("AppService", () => { expect(batchService.start).toBeCalledTimes(2); expect(counterService.start).toBeCalledTimes(2); expect(balancesCleanerService.start).toBeCalledTimes(2); + expect(tokenOffChainDataSaverService.start).toBeCalledTimes(2); await app.close(); }); diff --git a/packages/worker/src/app.service.ts b/packages/worker/src/app.service.ts index 5b841d954a..d6ec852c73 100644 --- a/packages/worker/src/app.service.ts +++ b/packages/worker/src/app.service.ts @@ -8,6 +8,7 @@ import { BlockService } from "./block"; import { BatchService } from "./batch"; import { CounterService } from "./counter"; import { BalancesCleanerService } from "./balance"; +import { TokenOffChainDataSaverService } from "./token/tokenOffChainData/tokenOffChainDataSaver.service"; import runMigrations from "./utils/runMigrations"; @Injectable() @@ -20,6 +21,7 @@ export class AppService implements OnModuleInit, OnModuleDestroy { private readonly blockService: BlockService, private readonly blocksRevertService: BlocksRevertService, private readonly balancesCleanerService: BalancesCleanerService, + private readonly tokenOffChainDataSaverService: TokenOffChainDataSaverService, private readonly dataSource: DataSource, private readonly configService: ConfigService ) { @@ -52,6 +54,7 @@ export class AppService implements OnModuleInit, OnModuleDestroy { const disableBatchesProcessing = this.configService.get("batches.disableBatchesProcessing"); const disableCountersProcessing = this.configService.get("counters.disableCountersProcessing"); const disableOldBalancesCleaner = this.configService.get("balances.disableOldBalancesCleaner"); + const enableTokenOffChainDataSaver = this.configService.get("tokens.enableTokenOffChainDataSaver"); const tasks = [this.blockService.start()]; if (!disableBatchesProcessing) { tasks.push(this.batchService.start()); @@ -62,6 +65,9 @@ export class AppService implements OnModuleInit, OnModuleDestroy { if (!disableOldBalancesCleaner) { tasks.push(this.balancesCleanerService.start()); } + if (enableTokenOffChainDataSaver) { + tasks.push(this.tokenOffChainDataSaverService.start()); + } return Promise.all(tasks); } @@ -71,6 +77,7 @@ export class AppService implements OnModuleInit, OnModuleDestroy { this.batchService.stop(), this.counterService.stop(), this.balancesCleanerService.stop(), + this.tokenOffChainDataSaverService.stop(), ]); } } diff --git a/packages/worker/src/config.spec.ts b/packages/worker/src/config.spec.ts index 0e01d47878..f4de4129bb 100644 --- a/packages/worker/src/config.spec.ts +++ b/packages/worker/src/config.spec.ts @@ -42,6 +42,11 @@ describe("config", () => { updateInterval: 30000, disableCountersProcessing: false, }, + tokens: { + enableTokenOffChainDataSaver: false, + updateTokenOffChainDataInterval: 86_400_000, + tokenOffChainDataMinLiquidityFilter: 1000_000, + }, metrics: { collectDbConnectionPoolMetricsInterval: 10000, collectBlocksToProcessMetricInterval: 10000, diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index f2bac617a5..f30fed7b75 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -19,6 +19,9 @@ export default () => { DISABLE_BALANCES_PROCESSING, DISABLE_OLD_BALANCES_CLEANER, DISABLE_BLOCKS_REVERT, + ENABLE_TOKEN_OFFCHAIN_DATA_SAVER, + UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL, + TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER, FROM_BLOCK, TO_BLOCK, } = process.env; @@ -53,6 +56,11 @@ export default () => { updateInterval: parseInt(COUNTERS_PROCESSING_POLLING_INTERVAL, 10) || 30000, disableCountersProcessing: DISABLE_COUNTERS_PROCESSING === "true", }, + tokens: { + enableTokenOffChainDataSaver: ENABLE_TOKEN_OFFCHAIN_DATA_SAVER === "true", + updateTokenOffChainDataInterval: parseInt(UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL, 10) || 86_400_000, + tokenOffChainDataMinLiquidityFilter: parseInt(TOKEN_OFFCHAIN_DATA_MIN_LIQUIDITY_FILTER, 10) || 1000_000, + }, metrics: { collectDbConnectionPoolMetricsInterval: parseInt(COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL, 10) || 10000, collectBlocksToProcessMetricInterval: parseInt(COLLECT_BLOCKS_TO_PROCESS_METRIC_INTERVAL, 10) || 10000, diff --git a/packages/worker/src/entities/token.entity.ts b/packages/worker/src/entities/token.entity.ts index 2d538bed0a..9cead7572c 100644 --- a/packages/worker/src/entities/token.entity.ts +++ b/packages/worker/src/entities/token.entity.ts @@ -14,11 +14,12 @@ export enum TokenType { @Entity({ name: "tokens" }) @Check(`"symbol" <> ''`) -@Index(["blockNumber", "logIndex"]) +@Index(["liquidity", "blockNumber", "logIndex"]) export class Token extends BaseEntity { @PrimaryColumn({ type: "bytea", transformer: hexTransformer }) public readonly l2Address: string; + @Index() @Column({ type: "bytea", nullable: true, transformer: hexTransformer }) public readonly l1Address?: string; @@ -51,4 +52,17 @@ export class Token extends BaseEntity { @Column({ type: "int" }) public readonly logIndex: number; + + @Column({ type: "double precision", nullable: true }) + public readonly usdPrice?: number; + + @Column({ type: "double precision", nullable: true }) + public readonly liquidity?: number; + + @Column({ nullable: true }) + public readonly iconURL?: string; + + @Index() + @Column({ type: "timestamp", nullable: true }) + public readonly offChainDataUpdatedAt?: Date; } diff --git a/packages/worker/src/migrations/1698729230516-AddTokenOffChainData.ts b/packages/worker/src/migrations/1698729230516-AddTokenOffChainData.ts new file mode 100644 index 0000000000..140b56ebe5 --- /dev/null +++ b/packages/worker/src/migrations/1698729230516-AddTokenOffChainData.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddTokenOffChainData1698729230516 implements MigrationInterface { + name = "AddTokenOffChainData1698729230516"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "tokens" ADD "usdPrice" double precision`); + await queryRunner.query(`ALTER TABLE "tokens" ADD "liquidity" double precision`); + await queryRunner.query(`ALTER TABLE "tokens" ADD "iconURL" character varying`); + await queryRunner.query(`ALTER TABLE "tokens" ADD "offChainDataUpdatedAt" TIMESTAMP`); + await queryRunner.query(`CREATE INDEX "IDX_eab106a9418a761805c415fdeb" ON "tokens" ("l1Address") `); + await queryRunner.query(`CREATE INDEX "IDX_f9d4a596adf95ca64ed98810fd" ON "tokens" ("offChainDataUpdatedAt") `); + await queryRunner.query( + `CREATE INDEX "IDX_c45d488f03a4724a1b8490068c" ON "tokens" ("liquidity", "blockNumber", "logIndex") ` + ); + await queryRunner.query(`DROP INDEX "public"."IDX_f1d168776e18becd1d1a5e594f"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_c45d488f03a4724a1b8490068c"`); + await queryRunner.query(`DROP INDEX "public"."IDX_f9d4a596adf95ca64ed98810fd"`); + await queryRunner.query(`DROP INDEX "public"."IDX_eab106a9418a761805c415fdeb"`); + await queryRunner.query(`ALTER TABLE "tokens" DROP COLUMN "offChainDataUpdatedAt"`); + await queryRunner.query(`ALTER TABLE "tokens" DROP COLUMN "iconURL"`); + await queryRunner.query(`ALTER TABLE "tokens" DROP COLUMN "liquidity"`); + await queryRunner.query(`ALTER TABLE "tokens" DROP COLUMN "usdPrice"`); + await queryRunner.query(`CREATE INDEX "IDX_f1d168776e18becd1d1a5e594f" ON "tokens" ("blockNumber", "logIndex") `); + } +} diff --git a/packages/worker/src/repositories/token.repository.spec.ts b/packages/worker/src/repositories/token.repository.spec.ts index ea76b99c85..8d5955961c 100644 --- a/packages/worker/src/repositories/token.repository.spec.ts +++ b/packages/worker/src/repositories/token.repository.spec.ts @@ -1,6 +1,6 @@ import { Test, TestingModule } from "@nestjs/testing"; import { mock } from "jest-mock-extended"; -import { EntityManager, SelectQueryBuilder, InsertQueryBuilder } from "typeorm"; +import { EntityManager, SelectQueryBuilder, InsertQueryBuilder, IsNull, Not } from "typeorm"; import { BaseRepository } from "./base.repository"; import { TokenRepository } from "./token.repository"; import { Token } from "../entities"; @@ -22,6 +22,13 @@ describe("TokenRepository", () => { entityManagerMock = mock({ createQueryBuilder: jest.fn().mockReturnValue(queryBuilderMock), + findOne: jest.fn().mockResolvedValue(null), + find: jest.fn().mockResolvedValue([ + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + }, + ]), + update: jest.fn().mockResolvedValue(null), }); unitOfWorkMock = mock({ getTransactionManager: jest.fn().mockReturnValue(entityManagerMock) }); @@ -79,4 +86,138 @@ describe("TokenRepository", () => { expect(insertQueryBuilderMock.execute).toBeCalledTimes(1); }); }); + + describe("getOffChainDataLastUpdatedAt", () => { + it("returns undefined when no offchain data update ever happened", async () => { + const result = await repository.getOffChainDataLastUpdatedAt(); + expect(entityManagerMock.findOne).toBeCalledWith(Token, { + where: { + offChainDataUpdatedAt: Not(IsNull()), + }, + select: { + offChainDataUpdatedAt: true, + }, + order: { + offChainDataUpdatedAt: "DESC", + }, + }); + expect(result).toBe(undefined); + }); + + it("queries last offchain data updated date", async () => { + const lastOffChainDataUpdatedAt = new Date(); + jest.spyOn(entityManagerMock, "findOne").mockResolvedValueOnce({ + offChainDataUpdatedAt: lastOffChainDataUpdatedAt, + }); + + const result = await repository.getOffChainDataLastUpdatedAt(); + expect(entityManagerMock.findOne).toBeCalledWith(Token, { + where: { + offChainDataUpdatedAt: Not(IsNull()), + }, + select: { + offChainDataUpdatedAt: true, + }, + order: { + offChainDataUpdatedAt: "DESC", + }, + }); + expect(result).toEqual(lastOffChainDataUpdatedAt); + }); + }); + + describe("getBridgedTokens", () => { + it("returns list of tokens having l1 address with l1Address field by default", async () => { + const result = await repository.getBridgedTokens(); + expect(entityManagerMock.find).toBeCalledWith(Token, { + where: { + l1Address: Not(IsNull()), + }, + select: { + l1Address: true, + }, + }); + expect(result).toEqual([ + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + }, + ]); + }); + + it("returns list of tokens having l1 address with specified fields", async () => { + jest.spyOn(entityManagerMock, "find").mockResolvedValueOnce([ + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + l2Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d2", + }, + ]); + const result = await repository.getBridgedTokens({ + l1Address: true, + l2Address: true, + }); + expect(entityManagerMock.find).toBeCalledWith(Token, { + where: { + l1Address: Not(IsNull()), + }, + select: { + l1Address: true, + l2Address: true, + }, + }); + expect(result).toEqual([ + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + l2Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d2", + }, + ]); + }); + }); + + describe("updateTokenOffChainData", () => { + it("updates token offchain data when iconURL is not provided", async () => { + const updatedAt = new Date(); + await repository.updateTokenOffChainData({ + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + liquidity: 1000000, + usdPrice: 55.89037747, + updatedAt, + }); + + expect(entityManagerMock.update).toBeCalledWith( + Token, + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + }, + { + liquidity: 1000000, + usdPrice: 55.89037747, + offChainDataUpdatedAt: updatedAt, + } + ); + }); + + it("updates token offchain data when iconURL is provided", async () => { + const updatedAt = new Date(); + await repository.updateTokenOffChainData({ + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + liquidity: 1000000, + usdPrice: 55.89037747, + updatedAt, + iconURL: "http://icon.com", + }); + + expect(entityManagerMock.update).toBeCalledWith( + Token, + { + l1Address: "0xD754fF5e8A6f257E162F72578A4Bb0493C0681d1", + }, + { + liquidity: 1000000, + usdPrice: 55.89037747, + offChainDataUpdatedAt: updatedAt, + iconURL: "http://icon.com", + } + ); + }); + }); }); diff --git a/packages/worker/src/repositories/token.repository.ts b/packages/worker/src/repositories/token.repository.ts index 01df2c9c8f..5b9e701096 100644 --- a/packages/worker/src/repositories/token.repository.ts +++ b/packages/worker/src/repositories/token.repository.ts @@ -1,5 +1,6 @@ import { Injectable } from "@nestjs/common"; import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity"; +import { IsNull, Not, FindOptionsSelect } from "typeorm"; import { Token } from "../entities"; import { BaseRepository } from "./base.repository"; import { UnitOfWork } from "../unitOfWork"; @@ -32,4 +33,61 @@ export class TokenRepository extends BaseRepository { `); await queryBuilder.execute(); } + + public async getOffChainDataLastUpdatedAt(): Promise { + const transactionManager = this.unitOfWork.getTransactionManager(); + const token = await transactionManager.findOne(this.entityTarget, { + where: { + offChainDataUpdatedAt: Not(IsNull()), + }, + select: { + offChainDataUpdatedAt: true, + }, + order: { + offChainDataUpdatedAt: "DESC", + }, + }); + return token?.offChainDataUpdatedAt; + } + + public async getBridgedTokens(fields: FindOptionsSelect = { l1Address: true }): Promise { + const transactionManager = this.unitOfWork.getTransactionManager(); + const tokens = await transactionManager.find(this.entityTarget, { + where: { + l1Address: Not(IsNull()), + }, + select: fields, + }); + return tokens; + } + + public async updateTokenOffChainData({ + l1Address, + liquidity, + usdPrice, + updatedAt, + iconURL, + }: { + l1Address: string; + liquidity: number; + usdPrice: number; + updatedAt: Date; + iconURL?: string; + }): Promise { + const transactionManager = this.unitOfWork.getTransactionManager(); + await transactionManager.update( + this.entityTarget, + { + l1Address, + }, + { + liquidity, + usdPrice, + offChainDataUpdatedAt: updatedAt, + ...(iconURL && { + iconURL, + }), + } + ); + } } diff --git a/packages/worker/src/token/token.service.ts b/packages/worker/src/token/token.service.ts index 5ed69b7515..45e6e513b4 100644 --- a/packages/worker/src/token/token.service.ts +++ b/packages/worker/src/token/token.service.ts @@ -1,7 +1,6 @@ import { types } from "zksync-web3"; import { Injectable, Logger } from "@nestjs/common"; import { InjectMetric } from "@willsoto/nestjs-prometheus"; -import { utils as ethersUtils } from "ethers"; import { Histogram } from "prom-client"; import { LogType, isLogOfType } from "../log/logType"; import { BlockchainService } from "../blockchain/blockchain.service"; @@ -25,7 +24,6 @@ export interface Token { @Injectable() export class TokenService { private readonly logger: Logger; - private readonly abiCoder: ethersUtils.AbiCoder; constructor( private readonly blockchainService: BlockchainService, @@ -33,7 +31,6 @@ export class TokenService { @InjectMetric(GET_TOKEN_INFO_DURATION_METRIC_NAME) private readonly getTokenInfoDurationMetric: Histogram ) { - this.abiCoder = new ethersUtils.AbiCoder(); this.logger = new Logger(TokenService.name); } diff --git a/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.spec.ts b/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.spec.ts new file mode 100644 index 0000000000..f12b29e9c5 --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.spec.ts @@ -0,0 +1,180 @@ +import { Test } from "@nestjs/testing"; +import { mock } from "jest-mock-extended"; +import { Logger } from "@nestjs/common"; +import { HttpService } from "@nestjs/axios"; +import { AxiosResponse, AxiosError } from "axios"; +import { setTimeout } from "timers/promises"; +import * as rxjs from "rxjs"; +import { PortalsFiTokenOffChainDataProvider } from "./portalsFiTokenOffChainDataProvider"; + +const MIN_TOKENS_LIQUIDITY_FILTER = 1000000; +const TOKENS_INFO_API_URL = "https://api.portals.fi/v2/tokens"; +const TOKENS_INFO_API_QUERY = `networks=ethereum&limit=250&sortBy=liquidity&minLiquidity=${MIN_TOKENS_LIQUIDITY_FILTER}&sortDirection=desc`; + +const providerTokensResponse = [ + { + address: "address1", + liquidity: 1000000, + price: 50.7887667, + image: "http://image1.com", + }, + { + address: "address2", + liquidity: 2000000, + price: 20.3454334, + image: "http://image2.com", + }, +]; + +jest.mock("timers/promises", () => ({ + setTimeout: jest.fn().mockResolvedValue(null), +})); + +describe("PortalsFiTokenOffChainDataProvider", () => { + let provider: PortalsFiTokenOffChainDataProvider; + let httpServiceMock: HttpService; + + beforeEach(async () => { + httpServiceMock = mock(); + const module = await Test.createTestingModule({ + providers: [ + PortalsFiTokenOffChainDataProvider, + { + provide: HttpService, + useValue: httpServiceMock, + }, + ], + }).compile(); + module.useLogger(mock()); + + provider = module.get(PortalsFiTokenOffChainDataProvider); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe("getTokensOffChainData", () => { + let pipeMock = jest.fn(); + + beforeEach(() => { + pipeMock = jest.fn(); + jest.spyOn(httpServiceMock, "get").mockReturnValue({ + pipe: pipeMock, + } as unknown as rxjs.Observable); + jest.spyOn(rxjs, "catchError").mockImplementation((callback) => callback as any); + }); + + it("returns empty array when fetching offchain data constantly fails", async () => { + pipeMock.mockImplementation((callback) => { + callback({ + stack: "error stack", + } as AxiosError); + }); + + const tokens = await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + expect(tokens).toEqual([]); + }); + + it("retries for 5 times each time doubling timeout when fetching offchain data constantly fails", async () => { + pipeMock.mockImplementation((callback) => { + callback({ + stack: "error stack", + response: { + data: "response data", + status: 500, + }, + } as AxiosError); + }); + + await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + expect(httpServiceMock.get).toBeCalledTimes(6); + expect(setTimeout).toBeCalledTimes(5); + }); + + it("fetches offchain tokens data with pagination and returns combined tokens list", async () => { + pipeMock + .mockReturnValueOnce( + new rxjs.Observable((subscriber) => { + subscriber.next({ + data: { + more: true, + tokens: [providerTokensResponse[0]], + }, + }); + }) + ) + .mockReturnValueOnce( + new rxjs.Observable((subscriber) => { + subscriber.next({ + data: { + more: false, + tokens: [providerTokensResponse[1]], + }, + }); + }) + ); + + const tokens = await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + expect(httpServiceMock.get).toBeCalledTimes(2); + expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0`); + expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=1`); + expect(tokens).toEqual([ + { + l1Address: "address1", + liquidity: 1000000, + usdPrice: 50.7887667, + iconURL: "http://image1.com", + }, + { + l1Address: "address2", + liquidity: 2000000, + usdPrice: 20.3454334, + iconURL: "http://image2.com", + }, + ]); + }); + + it("retries when provider API call fails", async () => { + pipeMock + .mockImplementationOnce((callback) => { + callback({ + stack: "error stack", + response: { + data: "response data", + status: 500, + }, + } as AxiosError); + }) + .mockReturnValueOnce( + new rxjs.Observable((subscriber) => { + subscriber.next({ + data: { + more: false, + tokens: providerTokensResponse, + }, + }); + }) + ); + + const tokens = await provider.getTokensOffChainData(MIN_TOKENS_LIQUIDITY_FILTER); + expect(httpServiceMock.get).toBeCalledTimes(2); + expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0`); + expect(httpServiceMock.get).toBeCalledWith(`${TOKENS_INFO_API_URL}?${TOKENS_INFO_API_QUERY}&page=0`); + expect(tokens).toEqual([ + { + l1Address: "address1", + liquidity: 1000000, + usdPrice: 50.7887667, + iconURL: "http://image1.com", + }, + { + l1Address: "address2", + liquidity: 2000000, + usdPrice: 20.3454334, + iconURL: "http://image2.com", + }, + ]); + }); + }); +}); diff --git a/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.ts b/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.ts new file mode 100644 index 0000000000..233b4798d7 --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/providers/portalsFiTokenOffChainDataProvider.ts @@ -0,0 +1,119 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { HttpService } from "@nestjs/axios"; +import { AxiosError } from "axios"; +import { setTimeout } from "timers/promises"; +import { catchError, firstValueFrom } from "rxjs"; +import { TokenOffChainDataProvider, ITokenOffChainData } from "../tokenOffChainDataProvider.abstract"; + +const TOKENS_INFO_API_URL = "https://api.portals.fi/v2/tokens"; +const API_INITIAL_RETRY_TIMEOUT = 5000; +const API_RETRY_ATTEMPTS = 5; + +interface ITokensOffChainDataPage { + hasMore: boolean; + tokens: ITokenOffChainData[]; +} + +interface ITokenOffChainDataProviderResponse { + address: string; + image?: string; + liquidity: number; + price: number; +} + +interface ITokensOffChainDataProviderResponse { + more: boolean; + tokens: ITokenOffChainDataProviderResponse[]; +} + +@Injectable() +export class PortalsFiTokenOffChainDataProvider implements TokenOffChainDataProvider { + private readonly logger: Logger; + + constructor(private readonly httpService: HttpService) { + this.logger = new Logger(PortalsFiTokenOffChainDataProvider.name); + } + + public async getTokensOffChainData(minLiquidity: number): Promise { + let page = 0; + let hasMore = true; + const tokens = []; + + while (hasMore) { + const tokensInfoPage = await this.getTokensOffChainDataPageRetryable({ page, minLiquidity }); + tokens.push(...tokensInfoPage.tokens); + page++; + hasMore = tokensInfoPage.hasMore; + } + + return tokens; + } + + private async getTokensOffChainDataPageRetryable({ + page, + minLiquidity, + retryAttempt = 0, + retryTimeout = API_INITIAL_RETRY_TIMEOUT, + }: { + page: number; + minLiquidity: number; + retryAttempt?: number; + retryTimeout?: number; + }): Promise { + try { + return await this.getTokensOffChainDataPage({ page, minLiquidity }); + } catch { + if (retryAttempt >= API_RETRY_ATTEMPTS) { + this.logger.error({ + message: `Failed to fetch tokens info at page=${page} after ${retryAttempt} retries`, + provider: PortalsFiTokenOffChainDataProvider.name, + }); + return { + hasMore: false, + tokens: [], + }; + } + await setTimeout(retryTimeout); + return this.getTokensOffChainDataPageRetryable({ + page, + minLiquidity, + retryAttempt: retryAttempt + 1, + retryTimeout: retryTimeout * 2, + }); + } + } + + private async getTokensOffChainDataPage({ + page, + minLiquidity, + }: { + page: number; + minLiquidity: number; + }): Promise { + const queryString = `networks=ethereum&limit=250&sortBy=liquidity&minLiquidity=${minLiquidity}&sortDirection=desc&page=${page}`; + + const { data } = await firstValueFrom<{ data: ITokensOffChainDataProviderResponse }>( + this.httpService.get(`${TOKENS_INFO_API_URL}?${queryString}`).pipe( + catchError((error: AxiosError) => { + this.logger.error({ + message: `Failed to fetch tokens info at page=${page}`, + stack: error.stack, + response: error.response?.data, + provider: PortalsFiTokenOffChainDataProvider.name, + }); + throw new Error(`Failed to fetch tokens info at page=${page}`); + }) + ) + ); + + return { + hasMore: data.more, + tokens: data.tokens.map((token) => ({ + l1Address: token.address, + liquidity: token.liquidity, + usdPrice: token.price, + iconURL: token.image, + })), + }; + } +} diff --git a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts new file mode 100644 index 0000000000..ff99d4cec7 --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataProvider.abstract.ts @@ -0,0 +1,10 @@ +export interface ITokenOffChainData { + l1Address: string; + liquidity: number; + usdPrice: number; + iconURL?: string; +} + +export abstract class TokenOffChainDataProvider { + abstract getTokensOffChainData: (minLiquidity: number) => Promise; +} diff --git a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts new file mode 100644 index 0000000000..2e810e3a86 --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.spec.ts @@ -0,0 +1,193 @@ +import { ConfigService } from "@nestjs/config"; +import { mock } from "jest-mock-extended"; +import waitFor from "../../utils/waitFor"; +import { TokenOffChainDataSaverService } from "./tokenOffChainDataSaver.service"; +import { TokenRepository } from "../../repositories/token.repository"; +import { TokenOffChainDataProvider } from "./tokenOffChainDataProvider.abstract"; +import { Token } from "../../entities"; + +jest.useFakeTimers().setSystemTime(new Date("2023-01-01T02:00:00.000Z")); + +jest.mock("@nestjs/common", () => { + return { + ...jest.requireActual("@nestjs/common"), + Logger: function () { + return { debug: jest.fn(), log: jest.fn(), error: jest.fn() }; + }, + }; +}); +jest.mock("../../utils/waitFor"); + +describe("TokenOffChainDataSaverService", () => { + const OFFCHAIN_DATA_UPDATE_INTERVAL = 86_400_000; + const MIN_LIQUIDITY_FILTER = 1000000; + const tokenOffChainDataMock = { + l1Address: "address", + liquidity: 100000, + usdPrice: 12.6789, + iconURL: "http://icon.com", + }; + let tokenRepositoryMock: TokenRepository; + let tokenOffChainDataProviderMock: TokenOffChainDataProvider; + let tokenOffChainDataSaverService: TokenOffChainDataSaverService; + + beforeEach(() => { + (waitFor as jest.Mock).mockResolvedValue(null); + tokenRepositoryMock = mock({ + getOffChainDataLastUpdatedAt: jest.fn().mockResolvedValue(null), + getBridgedTokens: jest.fn().mockResolvedValue([]), + updateTokenOffChainData: jest.fn().mockResolvedValue(null), + }); + tokenOffChainDataProviderMock = mock({ + getTokensOffChainData: jest.fn().mockResolvedValue([tokenOffChainDataMock]), + }); + + tokenOffChainDataSaverService = new TokenOffChainDataSaverService( + tokenRepositoryMock, + tokenOffChainDataProviderMock, + mock({ + get: jest + .fn() + .mockImplementation((key) => + key === "tokens.updateTokenOffChainDataInterval" ? OFFCHAIN_DATA_UPDATE_INTERVAL : MIN_LIQUIDITY_FILTER + ), + }) + ); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe("start", () => { + it("waits for specified update offchain data interval when fails to get last offchain data updated date", async () => { + jest.spyOn(tokenRepositoryMock, "getOffChainDataLastUpdatedAt").mockRejectedValueOnce(new Error("error")); + + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + const [conditionPredicate, waitTime] = (waitFor as jest.Mock).mock.calls[0]; + expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(1); + expect(waitFor).toBeCalledTimes(1); + expect(conditionPredicate()).toBeTruthy(); + expect(waitTime).toBe(OFFCHAIN_DATA_UPDATE_INTERVAL); + }); + + it("does not update offchain data when it was updated recently and waits for remaining time", async () => { + const lastUpdatedAt = new Date("2023-01-01T01:00:00.000Z"); + const remainingTimeToWaitForUpdate = + OFFCHAIN_DATA_UPDATE_INTERVAL - (new Date().getTime() - lastUpdatedAt.getTime()); + jest.spyOn(tokenRepositoryMock, "getOffChainDataLastUpdatedAt").mockResolvedValueOnce(lastUpdatedAt); + + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + const [conditionPredicate, waitTime] = (waitFor as jest.Mock).mock.calls[0]; + expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(1); + expect(waitFor).toBeCalledTimes(1); + expect(conditionPredicate()).toBeTruthy(); + expect(waitTime).toBe(remainingTimeToWaitForUpdate); + expect(tokenRepositoryMock.getBridgedTokens).not.toBeCalled(); + expect(tokenOffChainDataProviderMock.getTokensOffChainData).not.toBeCalled(); + }); + + it("does not update offchain data when there are no bridged token atm and waits for the next update", async () => { + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + const [conditionPredicate, waitTime] = (waitFor as jest.Mock).mock.calls[0]; + expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(1); + expect(waitFor).toBeCalledTimes(1); + expect(conditionPredicate()).toBeTruthy(); + expect(waitTime).toBe(OFFCHAIN_DATA_UPDATE_INTERVAL); + expect(tokenRepositoryMock.getBridgedTokens).toBeCalledTimes(1); + expect(tokenOffChainDataProviderMock.getTokensOffChainData).not.toBeCalled(); + }); + + it("updates offchain data when data is too old and there are bridged tokens to update", async () => { + const lastUpdatedAt = new Date("2022-01-01T01:00:00.000Z"); + jest.spyOn(tokenRepositoryMock, "getOffChainDataLastUpdatedAt").mockResolvedValueOnce(lastUpdatedAt); + jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "address" } as Token]); + + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + expect(tokenOffChainDataProviderMock.getTokensOffChainData).toBeCalledWith(MIN_LIQUIDITY_FILTER); + expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledTimes(1); + expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledWith({ + l1Address: "address", + liquidity: 100000, + usdPrice: 12.6789, + updatedAt: new Date(), + iconURL: "http://icon.com", + }); + }); + + it("updates offchain data when data was never updated and there are bridged tokens to update", async () => { + jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "address" } as Token]); + + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + expect(tokenOffChainDataProviderMock.getTokensOffChainData).toBeCalledWith(MIN_LIQUIDITY_FILTER); + expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledTimes(1); + expect(tokenRepositoryMock.updateTokenOffChainData).toHaveBeenCalledWith({ + l1Address: "address", + liquidity: 100000, + usdPrice: 12.6789, + updatedAt: new Date(), + iconURL: "http://icon.com", + }); + }); + + it("waits for specified timeout or worker stoppage after offchain data update", async () => { + jest.spyOn(tokenRepositoryMock, "getBridgedTokens").mockResolvedValueOnce([{ l1Address: "address" } as Token]); + + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + const [conditionPredicate, waitTime] = (waitFor as jest.Mock).mock.calls[0]; + expect(waitFor).toBeCalledTimes(1); + expect(conditionPredicate()).toBeTruthy(); + expect(waitTime).toBe(OFFCHAIN_DATA_UPDATE_INTERVAL); + }); + + it("starts the process only once when called multiple times", async () => { + tokenOffChainDataSaverService.start(); + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(1); + }); + + it("runs update process iteratively until stopped", async () => { + let secondIterationResolve: (value: unknown) => void; + const secondIterationPromise = new Promise((resolve) => (secondIterationResolve = resolve)); + + jest + .spyOn(tokenRepositoryMock, "getOffChainDataLastUpdatedAt") + .mockResolvedValueOnce(null) + .mockImplementationOnce(() => { + secondIterationResolve(null); + return Promise.resolve(null); + }) + .mockResolvedValueOnce(null); + + tokenOffChainDataSaverService.start(); + + await secondIterationPromise; + await tokenOffChainDataSaverService.stop(); + + expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(2); + }); + }); + + describe("stop", () => { + it("stops offchain data saver process", async () => { + tokenOffChainDataSaverService.start(); + await tokenOffChainDataSaverService.stop(); + + expect(tokenRepositoryMock.getOffChainDataLastUpdatedAt).toBeCalledTimes(1); + }); + }); +}); diff --git a/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts new file mode 100644 index 0000000000..5084c27a6f --- /dev/null +++ b/packages/worker/src/token/tokenOffChainData/tokenOffChainDataSaver.service.ts @@ -0,0 +1,88 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { Worker } from "../../common/worker"; +import waitFor from "../../utils/waitFor"; +import { TokenRepository } from "../../repositories/token.repository"; +import { TokenOffChainDataProvider } from "./tokenOffChainDataProvider.abstract"; + +const UPDATE_TOKENS_BATCH_SIZE = 100; + +@Injectable() +export class TokenOffChainDataSaverService extends Worker { + private readonly updateTokenOffChainDataInterval: number; + private readonly tokenOffChainDataMinLiquidityFilter: number; + private readonly logger: Logger; + + public constructor( + private readonly tokenRepository: TokenRepository, + private readonly tokenOffChainDataProvider: TokenOffChainDataProvider, + configService: ConfigService + ) { + super(); + this.updateTokenOffChainDataInterval = configService.get("tokens.updateTokenOffChainDataInterval"); + this.tokenOffChainDataMinLiquidityFilter = configService.get("tokens.tokenOffChainDataMinLiquidityFilter"); + this.logger = new Logger(TokenOffChainDataSaverService.name); + } + + protected async runProcess(): Promise { + let nextUpdateTimeout = this.updateTokenOffChainDataInterval; + try { + const lastUpdatedAt = await this.tokenRepository.getOffChainDataLastUpdatedAt(); + const now = new Date().getTime(); + const timeSinceLastUpdate = lastUpdatedAt ? now - lastUpdatedAt.getTime() : this.updateTokenOffChainDataInterval; + nextUpdateTimeout = + timeSinceLastUpdate >= this.updateTokenOffChainDataInterval + ? 0 + : this.updateTokenOffChainDataInterval - timeSinceLastUpdate; + + if (!nextUpdateTimeout) { + const bridgedTokens = await this.tokenRepository.getBridgedTokens(); + if (bridgedTokens.length) { + const tokensInfo = await this.tokenOffChainDataProvider.getTokensOffChainData( + this.tokenOffChainDataMinLiquidityFilter + ); + const tokensToUpdate = tokensInfo.filter((token) => + bridgedTokens.find((t) => t.l1Address === token.l1Address) + ); + const updatedAt = new Date(); + + let updateTokensTasks = []; + for (let i = 0; i < tokensToUpdate.length; i++) { + updateTokensTasks.push( + this.tokenRepository.updateTokenOffChainData({ + l1Address: tokensToUpdate[i].l1Address, + liquidity: tokensToUpdate[i].liquidity, + usdPrice: tokensToUpdate[i].usdPrice, + updatedAt, + iconURL: tokensToUpdate[i].iconURL, + }) + ); + if (updateTokensTasks.length === UPDATE_TOKENS_BATCH_SIZE || i === tokensToUpdate.length - 1) { + await Promise.all(updateTokensTasks); + updateTokensTasks = []; + } + } + + this.logger.log("Updated tokens offchain data", { + totalTokensUpdated: tokensToUpdate.length, + }); + } + + nextUpdateTimeout = this.updateTokenOffChainDataInterval; + } + } catch (err) { + this.logger.error({ + message: "Failed to update tokens offchain data", + originalError: err, + }); + nextUpdateTimeout = this.updateTokenOffChainDataInterval; + } + + await waitFor(() => !this.currentProcessPromise, nextUpdateTimeout); + if (!this.currentProcessPromise) { + return; + } + + return this.runProcess(); + } +}