feat: improve token transfers queries (#51)

# What ❔

Add new indexes and fields to improve token transfers queries

## Why ❔

Improving token transfers queries performance

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ +] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ +] Tests for the changes have been added / updated.

.github/pull_request_template.md

@@ -17,4 +17,3 @@
 - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs).
 - [ ] Tests for the changes have been added / updated.
 - [ ] Documentation comments have been added / updated.
-- [ ] Code has been formatted via `zk fmt` and `zk lint`.

packages/api/src/token/token.entity.ts

@@ -3,6 +3,7 @@ import { BaseEntity } from "../common/entities/base.entity";
 import { normalizeAddressTransformer } from "../common/transformers/normalizeAddress.transformer";
 
 export enum TokenType {
+  ETH = "ETH",
   ERC20 = "ERC20",
   ERC721 = "ERC721",
 }

packages/api/src/transfer/addressTransfer.entity.ts

@@ -1,12 +1,14 @@
 import { Entity, Column, Index, ManyToOne, JoinColumn, PrimaryColumn } from "typeorm";
 import { BaseEntity } from "../common/entities/base.entity";
 import { Transfer } from "./transfer.entity";
+import { TokenType } from "../token/token.entity";
 import { bigIntNumberTransformer } from "../common/transformers/bigIntNumber.transformer";
 import { normalizeAddressTransformer } from "../common/transformers/normalizeAddress.transformer";
 
 @Entity({ name: "addressTransfers" })
 @Index(["address", "isFeeOrRefund", "timestamp", "logIndex"])
-@Index(["address", "tokenAddress", "fields", "blockNumber", "logIndex"])
+@Index(["address", "tokenType", "blockNumber", "logIndex"])
+@Index(["address", "tokenAddress", "blockNumber", "logIndex"])
 export class AddressTransfer extends BaseEntity {
   @PrimaryColumn({ generated: true, type: "bigint" })
   public readonly number: number;
@@ -32,6 +34,9 @@ export class AddressTransfer extends BaseEntity {
   @Column({ type: "timestamp" })
   public readonly timestamp: Date;
 
+  @Column({ type: "enum", enum: TokenType, default: TokenType.ETH })
+  public readonly tokenType: TokenType;
+
   @Column({ type: "boolean" })
   public readonly isFeeOrRefund: boolean;
 

packages/api/src/transfer/transfer.entity.ts

@@ -1,6 +1,6 @@
 import { Entity, Column, Index, ManyToOne, JoinColumn, PrimaryColumn } from "typeorm";
 import { BaseEntity } from "../common/entities/base.entity";
-import { Token } from "../token/token.entity";
+import { Token, TokenType } from "../token/token.entity";
 import { normalizeAddressTransformer } from "../common/transformers/normalizeAddress.transformer";
 import { bigIntNumberTransformer } from "../common/transformers/bigIntNumber.transformer";
 import { hexTransformer } from "../common/transformers/hex.transformer";
@@ -19,7 +19,7 @@ export enum TransferType {
 @Index(["blockNumber", "logIndex"])
 @Index(["transactionHash", "timestamp", "logIndex"])
 @Index(["tokenAddress", "isFeeOrRefund", "timestamp", "logIndex"])
-@Index(["tokenAddress", "fields", "blockNumber", "logIndex"])
+@Index(["tokenAddress", "blockNumber", "logIndex"])
 export class Transfer extends BaseEntity {
   @PrimaryColumn({ generated: true, type: "bigint", select: false })
   public number: number;
@@ -64,6 +64,9 @@ export class Transfer extends BaseEntity {
   @Column({ type: "enum", enum: TransferType, default: TransferType.Transfer })
   public readonly type: TransferType;
 
+  @Column({ type: "enum", enum: TokenType, default: TokenType.ETH })
+  public readonly tokenType: TokenType;
+
   @Column({ type: "boolean", select: false })
   public readonly isFeeOrRefund: boolean;
 

packages/api/src/transfer/transfer.service.spec.ts

@@ -16,7 +16,6 @@ import { TokenType } from "../token/token.entity";
 import { AddressTransfer } from "./addressTransfer.entity";
 import * as utils from "../common/utils";
 import { normalizeAddressTransformer } from "../common/transformers/normalizeAddress.transformer";
-import { L2_ETH_TOKEN_ADDRESS } from "../common/constants";
 
 jest.mock("../common/utils");
 
@@ -228,26 +227,24 @@ describe("TransferService", () => {
       });
 
       describe("when token type is ERC20", () => {
-        it("adds ERC20 filter", async () => {
+        it("adds tokenAddress filter", async () => {
           await service.findTokenTransfers(filterOptions);
           expect(queryBuilderMock.where).toBeCalledTimes(1);
           expect(queryBuilderMock.where).toHaveBeenCalledWith({
             tokenAddress: filterOptions.tokenAddress,
-            fields: typeorm.IsNull(),
           });
         });
       });
 
       describe("when token type is ERC721", () => {
-        it("adds ERC721 filter", async () => {
+        it("adds tokenAddress filter", async () => {
           await service.findTokenTransfers({
             ...filterOptions,
             tokenType: TokenType.ERC721,
           });
           expect(queryBuilderMock.where).toBeCalledTimes(1);
           expect(queryBuilderMock.where).toHaveBeenCalledWith({
             tokenAddress: "tokenAddress",
-            fields: typeorm.Not(typeorm.IsNull()),
           });
         });
       });
@@ -345,13 +342,19 @@ describe("TransferService", () => {
       });
 
       describe("when token type is ERC20", () => {
-        it("adds ERC20 filter", async () => {
+        it("adds address and ERC20 filter", async () => {
           await service.findTokenTransfers(filterOptions);
           expect(addressTransfersQueryBuilderMock.where).toBeCalledTimes(1);
           expect(addressTransfersQueryBuilderMock.where).toHaveBeenCalledWith({
             address: filterOptions.address,
-            fields: typeorm.IsNull(),
           });
+          expect(addressTransfersQueryBuilderMock.andWhere).toBeCalledTimes(1);
+          expect(addressTransfersQueryBuilderMock.andWhere).toHaveBeenCalledWith(
+            `"addressTransfer"."tokenType" = :tokenType`,
+            {
+              tokenType: TokenType.ERC20,
+            }
+          );
         });
       });
 
@@ -364,8 +367,14 @@ describe("TransferService", () => {
           expect(addressTransfersQueryBuilderMock.where).toBeCalledTimes(1);
           expect(addressTransfersQueryBuilderMock.where).toHaveBeenCalledWith({
             address: filterOptions.address,
-            fields: typeorm.Not(typeorm.IsNull()),
           });
+          expect(addressTransfersQueryBuilderMock.andWhere).toBeCalledTimes(1);
+          expect(addressTransfersQueryBuilderMock.andWhere).toHaveBeenCalledWith(
+            `"addressTransfer"."tokenType" = :tokenType`,
+            {
+              tokenType: TokenType.ERC721,
+            }
+          );
         });
       });
 
@@ -385,19 +394,6 @@ describe("TransferService", () => {
         });
       });
 
-      describe("when token address is not specified", () => {
-        it("adds filter to exclude ETH token", async () => {
-          await service.findTokenTransfers(filterOptions);
-          expect(addressTransfersQueryBuilderMock.andWhere).toBeCalledTimes(1);
-          expect(addressTransfersQueryBuilderMock.andWhere).toHaveBeenCalledWith(
-            `"addressTransfer"."tokenAddress" != :tokenAddress`,
-            {
-              tokenAddress: normalizeAddressTransformer.to(L2_ETH_TOKEN_ADDRESS),
-            }
-          );
-        });
-      });
-
       it("joins transfers and tokens records to the address transfers", async () => {
         await service.findTokenTransfers(filterOptions);
         expect(addressTransfersQueryBuilderMock.leftJoinAndSelect).toBeCalledTimes(2);

packages/api/src/transfer/transfer.service.ts

@@ -1,9 +1,8 @@
 import { BadRequestException, Injectable } from "@nestjs/common";
 import { InjectRepository } from "@nestjs/typeorm";
-import { Repository, FindOperator, MoreThanOrEqual, LessThanOrEqual, IsNull, Not } from "typeorm";
+import { Repository, FindOperator, MoreThanOrEqual, LessThanOrEqual } from "typeorm";
 import { Pagination } from "nestjs-typeorm-paginate";
 import { paginate } from "../common/utils";
-import { L2_ETH_TOKEN_ADDRESS } from "../common/constants";
 import { IPaginationOptions, SortingOrder } from "../common/types";
 import { Transfer } from "./transfer.entity";
 import { TokenType } from "../token/token.entity";
@@ -105,15 +104,14 @@ export class TransferService {
       queryBuilder.addSelect(["transactionReceipt.gasUsed", "transactionReceipt.cumulativeGasUsed"]);
       queryBuilder.where({
         address,
-        fields: tokenType === TokenType.ERC721 ? Not(IsNull()) : IsNull(),
       });
       if (tokenAddress) {
         queryBuilder.andWhere(`"addressTransfer"."tokenAddress" = :tokenAddress`, {
           tokenAddress: normalizeAddressTransformer.to(tokenAddress),
         });
       } else {
-        queryBuilder.andWhere(`"addressTransfer"."tokenAddress" != :tokenAddress`, {
-          tokenAddress: normalizeAddressTransformer.to(L2_ETH_TOKEN_ADDRESS),
+        queryBuilder.andWhere(`"addressTransfer"."tokenType" = :tokenType`, {
+          tokenType,
         });
       }
       if (startBlock !== undefined) {
@@ -154,7 +152,6 @@ export class TransferService {
     queryBuilder.addSelect(["transactionReceipt.gasUsed", "transactionReceipt.cumulativeGasUsed"]);
     queryBuilder.where({
       tokenAddress,
-      fields: tokenType === TokenType.ERC721 ? Not(IsNull()) : IsNull(),
     });
     if (startBlock !== undefined) {
       queryBuilder.andWhere({

packages/api/test/address.e2e-spec.ts

@@ -12,7 +12,7 @@ import { Transaction } from "../src/transaction/entities/transaction.entity";
 import { AddressTransaction } from "../src/transaction/entities/addressTransaction.entity";
 import { TransactionReceipt } from "../src/transaction/entities/transactionReceipt.entity";
 import { Log } from "../src/log/log.entity";
-import { Token } from "../src/token/token.entity";
+import { Token, TokenType } from "../src/token/token.entity";
 import { BatchDetails } from "../src/batch/batchDetails.entity";
 import { Counter } from "../src/counter/counter.entity";
 import { Transfer, TransferType } from "../src/transfer/transfer.entity";
@@ -310,6 +310,7 @@ describe("AddressController (e2e)", () => {
         transactionIndex: i,
         timestamp: new Date("2022-11-21T18:16:51.000Z"),
         type,
+        tokenType: i % 2 ? TokenType.ERC20 : TokenType.ETH,
         tokenAddress:
           i % 2 ? "0x97d0a23f34e535e44df8ba84c53a0945cf0eeb67" : "0x000000000000000000000000000000000000800a",
         logIndex: i,
@@ -326,6 +327,7 @@ describe("AddressController (e2e)", () => {
           tokenAddress: transferSpec.tokenAddress,
           blockNumber: transferSpec.blockNumber,
           timestamp: transferSpec.timestamp,
+          tokenType: transferSpec.tokenType,
           isFeeOrRefund: transferSpec.isFeeOrRefund,
           logIndex: transferSpec.logIndex,
           isInternal: transferSpec.isInternal,
@@ -1029,6 +1031,7 @@ describe("AddressController (e2e)", () => {
               tokenAddress: "0x97d0a23F34E535e44dF8ba84c53A0945cF0eEb67",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e11",
               type: "mint",
+              tokenType: "ERC20",
               isInternal: false,
             },
             {
@@ -1048,6 +1051,7 @@ describe("AddressController (e2e)", () => {
               tokenAddress: "0x000000000000000000000000000000000000800A",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e11",
               type: "transfer",
+              tokenType: "ETH",
               isInternal: false,
             },
             {
@@ -1067,6 +1071,7 @@ describe("AddressController (e2e)", () => {
               tokenAddress: "0x97d0a23F34E535e44dF8ba84c53A0945cF0eEb67",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e11",
               type: "deposit",
+              tokenType: "ERC20",
               isInternal: false,
             },
           ])

packages/api/test/token.e2e-spec.ts

@@ -5,7 +5,7 @@ import { Repository } from "typeorm";
 import { getRepositoryToken } from "@nestjs/typeorm";
 import { AppModule } from "../src/app.module";
 import { configureApp } from "../src/configureApp";
-import { Token } from "../src/token/token.entity";
+import { Token, TokenType } from "../src/token/token.entity";
 import { BlockDetail } from "../src/block/blockDetail.entity";
 import { Transaction } from "../src/transaction/entities/transaction.entity";
 import { Transfer, TransferType } from "../src/transfer/transfer.entity";
@@ -108,6 +108,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: "1000",
         type: TransferType.Deposit,
+        tokenType: TokenType.ERC20,
         logIndex: transferIndex++,
         transactionIndex: 0,
         timestamp: "2022-11-21T18:16:51.000Z",
@@ -123,6 +124,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: "1000",
         type: TransferType.Fee,
+        tokenType: TokenType.ERC20,
         logIndex: transferIndex++,
         transactionIndex: 0,
         timestamp: "2022-11-21T18:16:51.000Z",
@@ -138,6 +140,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: "1000",
         type: TransferType.Mint,
+        tokenType: TokenType.ERC20,
         logIndex: transferIndex++,
         transactionIndex: 0,
         timestamp: "2022-11-21T18:16:51.000Z",
@@ -153,6 +156,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: "1000",
         type: TransferType.Transfer,
+        tokenType: TokenType.ERC20,
         logIndex: transferIndex++,
         transactionIndex: 0,
         timestamp: "2022-11-21T18:16:51.000Z",
@@ -168,6 +172,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: "1000",
         type: TransferType.Withdrawal,
+        tokenType: TokenType.ERC20,
         logIndex: transferIndex++,
         transactionIndex: 0,
         timestamp: "2022-11-21T18:16:51.000Z",
@@ -183,6 +188,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: undefined,
         type: TransferType.Mint,
+        tokenType: TokenType.ERC721,
         fields: { tokenId: "1" },
         logIndex: transferIndex++,
         transactionIndex: 0,
@@ -199,6 +205,7 @@ describe("TokenController (e2e)", () => {
         tokenAddress: "0xd754ff5e8a6f257e162f72578a4bb0493c068101",
         amount: "1000",
         type: TransferType.Refund,
+        tokenType: TokenType.ERC20,
         logIndex: transferIndex++,
         transactionIndex: 0,
         timestamp: "2022-11-21T18:16:51.000Z",
@@ -392,6 +399,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "transfer",
+              tokenType: "ERC20",
               isInternal: false,
             },
             {
@@ -411,6 +419,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "withdrawal",
+              tokenType: "ERC20",
               isInternal: false,
             },
             {
@@ -432,6 +441,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "mint",
+              tokenType: "ERC721",
               isInternal: false,
             },
             {
@@ -451,6 +461,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "deposit",
+              tokenType: "ERC20",
               isInternal: false,
             },
             {
@@ -470,6 +481,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "mint",
+              tokenType: "ERC20",
               isInternal: false,
             },
             {
@@ -489,6 +501,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "transfer",
+              tokenType: "ERC20",
               isInternal: false,
             },
             {
@@ -508,6 +521,7 @@ describe("TokenController (e2e)", () => {
               tokenAddress: "0xD754FF5E8a6F257E162f72578a4bB0493c068101",
               transactionHash: "0x8a008b8dbbc18035e56370abb820e736b705d68d6ac12b203603db8d9ea87e10",
               type: "withdrawal",
+              tokenType: "ERC20",
               isInternal: false,
             },
           ])