From acd2b35602fa2ce4efb71e8575b2775f15973fd6 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Fri, 4 Oct 2024 22:54:04 +0200 Subject: [PATCH 1/4] common: add pagination to tap collector Signed-off-by: Gustavo Inacio --- .../src/allocations/__tests__/tap.test.ts | 3 + .../src/allocations/tap-collector.ts | 189 ++++++++++++------ 2 files changed, 130 insertions(+), 62 deletions(-) diff --git a/packages/indexer-common/src/allocations/__tests__/tap.test.ts b/packages/indexer-common/src/allocations/__tests__/tap.test.ts index f0697d4d3..4ccc213d5 100644 --- a/packages/indexer-common/src/allocations/__tests__/tap.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/tap.test.ts @@ -101,6 +101,7 @@ const setupEach = async () => { _meta: { block: { timestamp: Date.now(), + hash: 'str', }, }, } @@ -460,6 +461,7 @@ describe('TAP', () => { return { transactions: [ { + id: 'test', allocationID: ALLOCATION_ID_2.toString().toLowerCase().replace('0x', ''), timestamp: redeemDateSecs, sender: { @@ -470,6 +472,7 @@ describe('TAP', () => { _meta: { block: { timestamp: nowSecs, + hash: 'test', }, }, } diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 02f6373c6..1919c56f7 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -27,7 +27,7 @@ import { import { BigNumber } from 'ethers' import pReduce from 'p-reduce' import { TAPSubgraph } from '../tap-subgraph' -import { NetworkSubgraph } from '../network-subgraph' +import { NetworkSubgraph, QueryResult } from '../network-subgraph' import gql from 'graphql-tag' // every 15 minutes @@ -65,16 +65,32 @@ interface RavWithAllocation { } export interface TapSubgraphResponse { - transactions: { - allocationID: string + transactions: TapTransaction[] + _meta: TapMeta +} + +interface TapMeta { + block: { timestamp: number - sender: { - id: string - } - }[] - _meta: { + hash: string + } +} + +interface TapTransaction { + id: string + allocationID: string + timestamp: number + sender: { + id: string + } +} + +export interface AllocationsResponse { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + allocations: any[] + meta: { block: { - timestamp: number + hash: string } } } @@ -202,12 +218,34 @@ export class TapCollector { const allocationIds: string[] = ravs.map((rav) => rav.getSignedRAV().rav.allocationId.toLowerCase(), ) + + let hash: string | undefined = undefined + let lastId = '' // eslint-disable-next-line @typescript-eslint/no-explicit-any - const returnedAllocations: any[] = ( - await this.networkSubgraph.query( + const returnedAllocations: any[] = [] + const pageSize = 1000 + + for (;;) { + const result = await this.networkSubgraph.query( gql` - query allocations($allocationIds: [String!]!) { - allocations(where: { id_in: $allocationIds }) { + query allocations( + $lastId: String! + $pageSize: Int! + $hash: String + $allocationIds: [String!]! + ) { + meta: _meta(block: { hash: $hash }) { + block { + number + hash + timestamp + } + } + allocations( + first: $pageSize + block: { hash: $hash } + where: { id_gt: $lastId, id_in: $allocationIds } + ) { id status subgraphDeployment { @@ -232,9 +270,19 @@ export class TapCollector { } } `, - { allocationIds }, + { allocationIds, lastId, pageSize, hash }, ) - ).data.allocations + if (!result.data) { + throw `There was an error while querying Network Subgraph. Errors: ${result.error}` + } + + if (result.data.allocations.length < pageSize) { + break + } + hash = result.data.meta.block.hash + lastId = result.data.allocations.slice(-1)[0].id + returnedAllocations.push(...result.data.allocations) + } if (returnedAllocations.length == 0) { this.logger.error( @@ -337,46 +385,73 @@ export class TapCollector { public async findTransactionsForRavs( ravs: ReceiptAggregateVoucher[], ): Promise { - const response = await this.tapSubgraph!.query( - gql` - query transactions( - $unfinalizedRavsAllocationIds: [String!]! - $senderAddresses: [String!]! - ) { - transactions( - where: { - type: "redeem" - allocationID_in: $unfinalizedRavsAllocationIds - sender_: { id_in: $senderAddresses } - } - ) { - allocationID - timestamp - sender { - id - } - } - _meta { - block { - timestamp + let meta: TapMeta | undefined = undefined + let lastId = '' + const transactions: TapTransaction[] = [] + const pageSize = 1000 + + for (;;) { + const hash = meta?.block?.hash + const result: QueryResult = + await this.tapSubgraph.query( + gql` + query transactions( + $lastId: String! + $pageSize: Int! + $hash: String + $unfinalizedRavsAllocationIds: [String!]! + $senderAddresses: [String!]! + ) { + transactions( + where: { + type: "redeem" + allocationID_in: $unfinalizedRavsAllocationIds + sender_: { id_in: $senderAddresses } + } + ) { + id + allocationID + timestamp + sender { + id + } + } + _meta { + block { + hash + timestamp + } + } } - } - } - `, - { - unfinalizedRavsAllocationIds: ravs.map((value) => - toAddress(value.allocationId).toLowerCase(), - ), - senderAddresses: ravs.map((value) => - toAddress(value.senderAddress).toLowerCase(), - ), - }, - ) - if (!response.data) { - throw `There was an error while querying Tap Subgraph. Errors: ${response.error}` + `, + { + lastId, + pageSize, + hash, + unfinalizedRavsAllocationIds: ravs.map((value) => + toAddress(value.allocationId).toLowerCase(), + ), + senderAddresses: ravs.map((value) => + toAddress(value.senderAddress).toLowerCase(), + ), + }, + ) + + if (!result.data) { + throw `There was an error while querying Tap Subgraph. Errors: ${result.error}` + } + meta = result.data._meta + if (result.data.transactions.length < pageSize) { + break + } + lastId = result.data.transactions.slice(-1)[0].id + transactions.push(...result.data.transactions) } - return response.data + return { + transactions, + _meta: meta!, + } } // for every allocation_id of this list that contains the redeemedAt less than the current @@ -438,16 +513,6 @@ export class TapCollector { function: 'submitRAVs()', ravsToSubmit: signedRavs.length, }) - if (!this.tapContracts) { - logger.error( - `Undefined escrow contracts, but this shouldn't happen as RAV process is only triggered when escrow is provided. \n - If this error is encountered please report and oepn an issue at https://github.com/graphprotocol/indexer/issues`, - { - signedRavs, - }, - ) - return - } const escrow = this.tapContracts logger.info(`Redeem last RAVs on chain individually`, { From 1eb15dc586252a0b58cd4fb02f7ab35a35a8f48b Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 10 Oct 2024 15:38:23 +0200 Subject: [PATCH 2/4] common: fix paginated tap queries Signed-off-by: Gustavo Inacio --- .../src/allocations/tap-collector.ts | 46 ++++++++++++------- packages/indexer-common/src/network.ts | 2 +- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 1919c56f7..99a7873a0 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -33,6 +33,8 @@ import gql from 'graphql-tag' // every 15 minutes const RAV_CHECK_INTERVAL_MS = 900_000 +const PAGE_SIZE = 1000 + interface RavMetrics { ravRedeemsSuccess: Counter ravRedeemsInvalid: Counter @@ -111,7 +113,7 @@ export class TapCollector { // eslint-disable-next-line @typescript-eslint/no-empty-function -- Private constructor to prevent direct instantiation private constructor() {} - public static async create({ + public static create({ logger, metrics, transactionManager, @@ -121,7 +123,7 @@ export class TapCollector { networkSpecification, tapSubgraph, networkSubgraph, - }: TapCollectorOptions): Promise { + }: TapCollectorOptions): TapCollector { const collector = new TapCollector() collector.logger = logger.child({ component: 'AllocationReceiptCollector' }) collector.metrics = registerReceiptMetrics( @@ -219,11 +221,10 @@ export class TapCollector { rav.getSignedRAV().rav.allocationId.toLowerCase(), ) - let hash: string | undefined = undefined + let block: { hash: string } | undefined = undefined let lastId = '' // eslint-disable-next-line @typescript-eslint/no-explicit-any const returnedAllocations: any[] = [] - const pageSize = 1000 for (;;) { const result = await this.networkSubgraph.query( @@ -231,10 +232,10 @@ export class TapCollector { query allocations( $lastId: String! $pageSize: Int! - $hash: String + $block: Block_height $allocationIds: [String!]! ) { - meta: _meta(block: { hash: $hash }) { + meta: _meta(block: $block) { block { number hash @@ -243,7 +244,9 @@ export class TapCollector { } allocations( first: $pageSize - block: { hash: $hash } + block: $block + orderBy: id, + orderDirection: asc, where: { id_gt: $lastId, id_in: $allocationIds } ) { id @@ -270,16 +273,17 @@ export class TapCollector { } } `, - { allocationIds, lastId, pageSize, hash }, + { allocationIds, lastId, pageSize: PAGE_SIZE, block }, ) + console.log("called query!") if (!result.data) { throw `There was an error while querying Network Subgraph. Errors: ${result.error}` } - if (result.data.allocations.length < pageSize) { + if (result.data.allocations.length < PAGE_SIZE) { break } - hash = result.data.meta.block.hash + block = { hash: result.data.meta.block.hash } lastId = result.data.allocations.slice(-1)[0].id returnedAllocations.push(...result.data.allocations) } @@ -388,22 +392,32 @@ export class TapCollector { let meta: TapMeta | undefined = undefined let lastId = '' const transactions: TapTransaction[] = [] - const pageSize = 1000 for (;;) { - const hash = meta?.block?.hash + let block: { hash: string } | undefined = undefined + if (meta?.block?.hash) { + block = { + hash: meta?.block?.hash, + } + } + const result: QueryResult = await this.tapSubgraph.query( gql` query transactions( $lastId: String! $pageSize: Int! - $hash: String + $block: Block_height $unfinalizedRavsAllocationIds: [String!]! $senderAddresses: [String!]! ) { transactions( + first: $pageSize + block: $block + orderBy: id, + orderDirection: asc, where: { + id_gt: $lastId type: "redeem" allocationID_in: $unfinalizedRavsAllocationIds sender_: { id_in: $senderAddresses } @@ -426,8 +440,8 @@ export class TapCollector { `, { lastId, - pageSize, - hash, + pageSize: PAGE_SIZE, + block, unfinalizedRavsAllocationIds: ravs.map((value) => toAddress(value.allocationId).toLowerCase(), ), @@ -441,7 +455,7 @@ export class TapCollector { throw `There was an error while querying Tap Subgraph. Errors: ${result.error}` } meta = result.data._meta - if (result.data.transactions.length < pageSize) { + if (result.data.transactions.length < PAGE_SIZE) { break } lastId = result.data.transactions.slice(-1)[0].id diff --git a/packages/indexer-common/src/network.ts b/packages/indexer-common/src/network.ts index 29f8fecb6..d955c494d 100644 --- a/packages/indexer-common/src/network.ts +++ b/packages/indexer-common/src/network.ts @@ -288,7 +288,7 @@ export class Network { // -------------------------------------------------------------------------------- let tapCollector: TapCollector | undefined = undefined if (tapContracts && tapSubgraph) { - tapCollector = await TapCollector.create({ + tapCollector = TapCollector.create({ logger, metrics, transactionManager: transactionManager, From 0c6589b589e03cee3283608936b5fb5b9cd94ed7 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 10 Oct 2024 15:39:06 +0200 Subject: [PATCH 3/4] common: add tests for pagination Signed-off-by: Gustavo Inacio --- .../__tests__/tap-pagination.test.ts | 217 ++++++++++++++++++ .../__tests__/validate-queries.test.ts | 87 +++++++ .../src/allocations/tap-collector.ts | 17 +- .../__tests__/helpers.test.ts | 2 +- 4 files changed, 313 insertions(+), 10 deletions(-) create mode 100644 packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts create mode 100644 packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts diff --git a/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts new file mode 100644 index 000000000..657564954 --- /dev/null +++ b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts @@ -0,0 +1,217 @@ +import { Address, Eventual, createLogger, createMetrics } from '@graphprotocol/common-ts' +import { + Allocation, + AllocationsResponse, + NetworkSubgraph, + QueryFeeModels, + QueryResult, + TapCollector, + TapSubgraphResponse, + TapTransaction, + TransactionManager, +} from '@graphprotocol/indexer-common' +import { NetworkContracts as TapContracts } from '@semiotic-labs/tap-contracts-bindings' +import { TAPSubgraph } from '../../tap-subgraph' +import { NetworkSpecification } from 'indexer-common/src/network-specification' +import { createMockAllocation } from '../../indexer-management/__tests__/helpers.test' +import { getContractAddress } from 'ethers/lib/utils' + +const timeout = 30_000 + +// mock allocation subgraph responses +// +// firstPage // 1000 +// secondPage // 1000 +// thirdPage // 999 +const allocations: Allocation[] = [] +const from = '0x8ba1f109551bD432803012645Ac136ddd64DBA72' + +for (let i = 0; i < 2999; i++) { + const mockAllocation = createMockAllocation() + allocations.push({ + ...mockAllocation, + id: getContractAddress({ from, nonce: i }) as Address, + }) +} + +// mock transactions subgraph response +// +// firstPage // 1000 +// secondPage // 1000 +const transactions: TapTransaction[] = [] +for (let i = 0; i < 2000; i++) { + transactions.push({ + id: i.toString(), + sender: { id: 'sender' }, + allocationID: 'allocation id', + timestamp: i, + }) +} + +// Make global Jest variables available +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const __LOG_LEVEL__: never +let tapCollector: TapCollector + +function paginateArray( + array: T[], + getId: (item: T) => string, + pageSize: number, + lastId?: string, +): T[] { + // Sort the array by ID to ensure consistent pagination. + array.sort((a, b) => getId(a).localeCompare(getId(b))) + + // Find the index of the item with the given lastId. + let startIndex = 0 + if (lastId) { + startIndex = array.findIndex((item) => getId(item) === lastId) + 1 + } + + // Slice the array to return only the requested page size. + return array.slice(startIndex, startIndex + pageSize) +} + +const mockQueryTapSubgraph = jest + .fn() + .mockImplementation(async (_, variables): Promise> => { + const pageSize: number = variables.pageSize + const lastId: string | undefined = variables.lastId + + const paginatedTransactions = paginateArray( + transactions, + (tx) => tx.id, + pageSize, + lastId, + ) + + return { + data: { + transactions: paginatedTransactions, + _meta: { + block: { + hash: 'blockhash', + timestamp: 100000, + }, + }, + }, + } + }) + +const mockQueryNetworkSubgraph = jest + .fn() + .mockImplementation(async (_, variables): Promise> => { + const pageSize: number = variables.pageSize + const lastId: string | undefined = variables.lastId + + const paginatedAllocations = paginateArray( + allocations, + (allocation) => allocation.id, + pageSize, + lastId, + ) + + return { + data: { + allocations: paginatedAllocations, + meta: { + block: { + hash: 'blockhash', + }, + }, + }, + } + }) + +jest.spyOn(TapCollector.prototype, 'startRAVProcessing').mockImplementation() +const setup = () => { + const logger = createLogger({ + name: 'Indexer API Client', + async: false, + level: __LOG_LEVEL__ ?? 'error', + }) + const metrics = createMetrics() + // Clearing the registry prevents duplicate metric registration in the default registry. + metrics.registry.clear() + const transactionManager = null as unknown as TransactionManager + const models = null as unknown as QueryFeeModels + const tapContracts = null as unknown as TapContracts + const allocations = null as unknown as Eventual + const networkSpecification = { + indexerOptions: { voucherRedemptionThreshold: 0, finalityTime: 0 }, + networkIdentifier: 'test', + } as unknown as NetworkSpecification + + const tapSubgraph = { + query: mockQueryTapSubgraph, + } as unknown as TAPSubgraph + const networkSubgraph = { + query: mockQueryNetworkSubgraph, + } as unknown as NetworkSubgraph + + tapCollector = TapCollector.create({ + logger, + metrics, + transactionManager, + models, + tapContracts, + allocations, + networkSpecification, + + networkSubgraph, + tapSubgraph, + }) +} + +describe('TAP Pagination', () => { + beforeAll(setup, timeout) + test( + 'test `getAllocationsfromAllocationIds` pagination', + async () => { + { + const allocations = await tapCollector['getAllocationsfromAllocationIds']([]) + expect(mockQueryNetworkSubgraph).toBeCalledTimes(3) + expect(allocations.length).toEqual(2999) + } + mockQueryNetworkSubgraph.mockClear() + + const mockAllocation = createMockAllocation() + allocations.push({ + ...mockAllocation, + id: getContractAddress({ from, nonce: 3000 }) as Address, + }) + { + const allocations = await tapCollector['getAllocationsfromAllocationIds']([]) + expect(mockQueryNetworkSubgraph).toBeCalledTimes(4) + expect(allocations.length).toEqual(3000) + } + }, + timeout, + ) + test( + 'test `findTransactionsForRavs` pagination', + async () => { + { + const transactionsResponse = await tapCollector['findTransactionsForRavs']([]) + expect(mockQueryTapSubgraph).toBeCalledTimes(3) + expect(transactionsResponse.transactions.length).toEqual(2000) + } + + mockQueryTapSubgraph.mockClear() + for (let i = 0; i < 500; i++) { + transactions.push({ + id: i.toString(), + sender: { id: 'sender' }, + allocationID: 'allocation id', + timestamp: i, + }) + } + { + const transactionsResponse = await tapCollector['findTransactionsForRavs']([]) + expect(mockQueryTapSubgraph).toBeCalledTimes(3) + expect(transactionsResponse.transactions.length).toEqual(2500) + } + }, + timeout, + ) +}) diff --git a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts new file mode 100644 index 000000000..798858167 --- /dev/null +++ b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts @@ -0,0 +1,87 @@ +import { + defineQueryFeeModels, + GraphNode, + Network, + QueryFeeModels, + TapCollector, +} from '@graphprotocol/indexer-common' +import { + connectDatabase, + createLogger, + createMetrics, + Logger, + Metrics, +} from '@graphprotocol/common-ts' +import { testNetworkSpecification } from '../../indexer-management/__tests__/util' +import { Sequelize } from 'sequelize' + +// Make global Jest variables available +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const __DATABASE__: any +declare const __LOG_LEVEL__: never +let logger: Logger +let tapCollector: TapCollector +let metrics: Metrics +let queryFeeModels: QueryFeeModels +let sequelize: Sequelize +const timeout = 30000 + +const setup = async () => { + logger = createLogger({ + name: 'Indexer API Client', + async: false, + level: __LOG_LEVEL__ ?? 'error', + }) + metrics = createMetrics() + // Clearing the registry prevents duplicate metric registration in the default registry. + metrics.registry.clear() + sequelize = await connectDatabase(__DATABASE__) + queryFeeModels = defineQueryFeeModels(sequelize) + sequelize = await sequelize.sync({ force: true }) + + const graphNode = new GraphNode( + logger, + 'https://test-admin-endpoint.xyz', + 'https://test-query-endpoint.xyz', + 'https://test-status-endpoint.xyz', + ) + + const network = await Network.create( + logger, + testNetworkSpecification, + queryFeeModels, + graphNode, + metrics, + ) + tapCollector = network.tapCollector! +} + +jest.spyOn(TapCollector.prototype, 'startRAVProcessing').mockImplementation() +describe('Validate TAP queries', () => { + beforeAll(setup, timeout) + + test( + 'test `getAllocationsfromAllocationIds` query is valid', + async () => { + const mockedFunc = jest.spyOn(tapCollector.networkSubgraph, 'query') + const result = await tapCollector['getAllocationsfromAllocationIds']([]) + expect(result).toEqual([]) + // this subgraph is in an eventual + // we check if it was called more than 0 times + expect(mockedFunc).toBeCalled() + }, + timeout, + ) + + test( + 'test `findTransactionsForRavs` query is valid', + async () => { + const mockedFunc = jest.spyOn(tapCollector.tapSubgraph, 'query') + const result = await tapCollector['findTransactionsForRavs']([]) + expect(result.transactions).toEqual([]) + expect(result._meta.block.hash.length).toEqual(66) + expect(mockedFunc).toBeCalledTimes(1) + }, + timeout, + ) +}) diff --git a/packages/indexer-common/src/allocations/tap-collector.ts b/packages/indexer-common/src/allocations/tap-collector.ts index 99a7873a0..1b0d497d4 100644 --- a/packages/indexer-common/src/allocations/tap-collector.ts +++ b/packages/indexer-common/src/allocations/tap-collector.ts @@ -78,7 +78,7 @@ interface TapMeta { } } -interface TapTransaction { +export interface TapTransaction { id: string allocationID: string timestamp: number @@ -245,8 +245,8 @@ export class TapCollector { allocations( first: $pageSize block: $block - orderBy: id, - orderDirection: asc, + orderBy: id + orderDirection: asc where: { id_gt: $lastId, id_in: $allocationIds } ) { id @@ -275,17 +275,16 @@ export class TapCollector { `, { allocationIds, lastId, pageSize: PAGE_SIZE, block }, ) - console.log("called query!") if (!result.data) { throw `There was an error while querying Network Subgraph. Errors: ${result.error}` } + returnedAllocations.push(...result.data.allocations) + block = { hash: result.data.meta.block.hash } if (result.data.allocations.length < PAGE_SIZE) { break } - block = { hash: result.data.meta.block.hash } lastId = result.data.allocations.slice(-1)[0].id - returnedAllocations.push(...result.data.allocations) } if (returnedAllocations.length == 0) { @@ -414,8 +413,8 @@ export class TapCollector { transactions( first: $pageSize block: $block - orderBy: id, - orderDirection: asc, + orderBy: id + orderDirection: asc where: { id_gt: $lastId type: "redeem" @@ -455,11 +454,11 @@ export class TapCollector { throw `There was an error while querying Tap Subgraph. Errors: ${result.error}` } meta = result.data._meta + transactions.push(...result.data.transactions) if (result.data.transactions.length < PAGE_SIZE) { break } lastId = result.data.transactions.slice(-1)[0].id - transactions.push(...result.data.transactions) } return { diff --git a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts index 62c7ca595..a2358926e 100644 --- a/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts +++ b/packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts @@ -122,7 +122,7 @@ const setupMonitor = async () => { ) } -const createMockAllocation = (): Allocation => { +export const createMockAllocation = (): Allocation => { const mockDeployment = { id: new SubgraphDeploymentID('QmcpeU4pZxzKB9TJ6fzH6PyZi9h8PJ6pG1c4izb9VAakJq'), deniedAt: 0, From 43111157563c9e8d12ba6f5f9882b9a5601a5da7 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 10 Oct 2024 18:55:48 +0200 Subject: [PATCH 4/4] common: fix mocked tests Signed-off-by: Gustavo Inacio --- .../__tests__/tap-pagination.test.ts | 168 ++++++++++-------- .../__tests__/validate-queries.test.ts | 3 + packages/indexer-common/src/tap-subgraph.ts | 2 + 3 files changed, 95 insertions(+), 78 deletions(-) diff --git a/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts index 657564954..abfeee144 100644 --- a/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/tap-pagination.test.ts @@ -72,56 +72,10 @@ function paginateArray( return array.slice(startIndex, startIndex + pageSize) } -const mockQueryTapSubgraph = jest - .fn() - .mockImplementation(async (_, variables): Promise> => { - const pageSize: number = variables.pageSize - const lastId: string | undefined = variables.lastId - - const paginatedTransactions = paginateArray( - transactions, - (tx) => tx.id, - pageSize, - lastId, - ) - - return { - data: { - transactions: paginatedTransactions, - _meta: { - block: { - hash: 'blockhash', - timestamp: 100000, - }, - }, - }, - } - }) - -const mockQueryNetworkSubgraph = jest - .fn() - .mockImplementation(async (_, variables): Promise> => { - const pageSize: number = variables.pageSize - const lastId: string | undefined = variables.lastId - - const paginatedAllocations = paginateArray( - allocations, - (allocation) => allocation.id, - pageSize, - lastId, - ) - - return { - data: { - allocations: paginatedAllocations, - meta: { - block: { - hash: 'blockhash', - }, - }, - }, - } - }) +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let mockQueryNetworkSubgraph: jest.Mock +// eslint-disable-next-line @typescript-eslint/no-explicit-any +let mockQueryTapSubgraph: jest.Mock jest.spyOn(TapCollector.prototype, 'startRAVProcessing').mockImplementation() const setup = () => { @@ -133,34 +87,92 @@ const setup = () => { const metrics = createMetrics() // Clearing the registry prevents duplicate metric registration in the default registry. metrics.registry.clear() - const transactionManager = null as unknown as TransactionManager - const models = null as unknown as QueryFeeModels - const tapContracts = null as unknown as TapContracts - const allocations = null as unknown as Eventual - const networkSpecification = { - indexerOptions: { voucherRedemptionThreshold: 0, finalityTime: 0 }, - networkIdentifier: 'test', - } as unknown as NetworkSpecification - - const tapSubgraph = { - query: mockQueryTapSubgraph, - } as unknown as TAPSubgraph - const networkSubgraph = { - query: mockQueryNetworkSubgraph, - } as unknown as NetworkSubgraph - - tapCollector = TapCollector.create({ - logger, - metrics, - transactionManager, - models, - tapContracts, - allocations, - networkSpecification, - - networkSubgraph, - tapSubgraph, - }) + + mockQueryTapSubgraph = jest + .fn() + .mockImplementation( + async (_, variables): Promise> => { + console.log('MOCKING IMPLEMENTATION FOR TAP SUBGRAPH') + const pageSize: number = variables.pageSize + const lastId: string | undefined = variables.lastId + + const paginatedTransactions = paginateArray( + transactions, + (tx) => tx.id, + pageSize, + lastId, + ) + + return { + data: { + transactions: paginatedTransactions, + _meta: { + block: { + hash: 'blockhash', + timestamp: 100000, + }, + }, + }, + } + }, + ) + + mockQueryNetworkSubgraph = jest + .fn() + .mockImplementation( + async (_, variables): Promise> => { + const pageSize: number = variables.pageSize + const lastId: string | undefined = variables.lastId + + const paginatedAllocations = paginateArray( + allocations, + (allocation) => allocation.id, + pageSize, + lastId, + ) + + return { + data: { + allocations: paginatedAllocations, + meta: { + block: { + hash: 'blockhash', + }, + }, + }, + } + }, + ) + { + const transactionManager = null as unknown as TransactionManager + const models = null as unknown as QueryFeeModels + const tapContracts = null as unknown as TapContracts + const allocations = null as unknown as Eventual + const networkSpecification = { + indexerOptions: { voucherRedemptionThreshold: 0, finalityTime: 0 }, + networkIdentifier: 'test', + } as unknown as NetworkSpecification + + const tapSubgraph = { + query: mockQueryTapSubgraph, + } as unknown as TAPSubgraph + const networkSubgraph = { + query: mockQueryNetworkSubgraph, + } as unknown as NetworkSubgraph + + tapCollector = TapCollector.create({ + logger, + metrics, + transactionManager, + models, + tapContracts, + allocations, + networkSpecification, + + networkSubgraph, + tapSubgraph, + }) + } } describe('TAP Pagination', () => { diff --git a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts index 798858167..b6836867f 100644 --- a/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts +++ b/packages/indexer-common/src/allocations/__tests__/validate-queries.test.ts @@ -69,6 +69,7 @@ describe('Validate TAP queries', () => { // this subgraph is in an eventual // we check if it was called more than 0 times expect(mockedFunc).toBeCalled() + mockedFunc.mockReset() }, timeout, ) @@ -77,10 +78,12 @@ describe('Validate TAP queries', () => { 'test `findTransactionsForRavs` query is valid', async () => { const mockedFunc = jest.spyOn(tapCollector.tapSubgraph, 'query') + const result = await tapCollector['findTransactionsForRavs']([]) expect(result.transactions).toEqual([]) expect(result._meta.block.hash.length).toEqual(66) expect(mockedFunc).toBeCalledTimes(1) + mockedFunc.mockReset() }, timeout, ) diff --git a/packages/indexer-common/src/tap-subgraph.ts b/packages/indexer-common/src/tap-subgraph.ts index 8e0f2da9f..0264c18f8 100644 --- a/packages/indexer-common/src/tap-subgraph.ts +++ b/packages/indexer-common/src/tap-subgraph.ts @@ -43,6 +43,8 @@ export class TAPSubgraph { // eslint-disable-next-line @typescript-eslint/no-explicit-any variables?: Record, ): Promise> { + // magic solution for jest tests https://stackoverflow.com/questions/69976411/jest-tlswrap-open-handle-error-using-simple-node-postgres-pool-query-fixed-wit/70012434#70012434 + await Promise.resolve(process.nextTick(Boolean)) const response = await this.endpointClient.post('', { query: print(query), variables,