From fb7c486d784d2ac4b39bf9e2ccb3033970531fa3 Mon Sep 17 00:00:00 2001 From: Filippo Date: Wed, 10 Jul 2024 13:54:04 +0200 Subject: [PATCH] 186-Store-`PoolSnapshot`-data-on-epoch-close-linked-to-`Epoch` (#223) feat: snapshotPeriod entity introduced feat: execute poolSnapshots after epoch execution BREAKING CHANGE: `periodStart` deprecated on Snapshots. Substituted by `period` foreign key --- schema.graphql | 23 ++++++-- src/helpers/stateSnapshot.test.ts | 31 +++++++++-- src/helpers/stateSnapshot.ts | 36 ++++++++++-- src/mappings/handlers/blockHandlers.ts | 55 +++++++++++++------ src/mappings/handlers/ethHandlers.ts | 17 +++++- src/mappings/handlers/poolsHandlers.ts | 4 ++ src/mappings/services/assetService.ts | 2 +- .../services/snapshotPeriodService.ts | 12 ++++ src/mappings/services/trancheService.ts | 4 +- 9 files changed, 148 insertions(+), 36 deletions(-) create mode 100644 src/mappings/services/snapshotPeriodService.ts diff --git a/schema.graphql b/schema.graphql index 11e467b4..01eea5d8 100644 --- a/schema.graphql +++ b/schema.graphql @@ -3,6 +3,20 @@ type Timekeeper @entity { lastPeriodStart: Date! } +type SnapshotPeriod @entity { + id: ID! + start: Date! @index + day: Int! @index + weekDay: Int! @index + month: Int! @index + year: Int! @index + + poolSnapshots: [PoolSnapshot] @derivedFrom(field: "period") + trancheSnapshots: [TrancheSnapshot] @derivedFrom(field: "period") + assetSnapshots: [AssetSnapshot] @derivedFrom(field: "period") + poolFeeSnapshots: [PoolFeeSnapshot] @derivedFrom(field: "period") +} + type Pool @entity { id: ID! #poolId # It's not possible to simply retrieve all entities, but it is supported @@ -92,7 +106,8 @@ type PoolSnapshot @entity { timestamp: Date! blockNumber: Int! - periodStart: Date! @index + period: SnapshotPeriod @index + epoch: Epoch @index normalizedNAV: BigInt # netAssetValue, normalized to 18 decimals @@ -191,7 +206,7 @@ type TrancheSnapshot @entity { timestamp: Date! blockNumber: Int! - periodStart: Date! @index + period: SnapshotPeriod @index tokenSupply: BigInt tokenPrice: BigInt @@ -479,7 +494,7 @@ type AssetSnapshot @entity { timestamp: Date! blockNumber: Int! - periodStart: Date! @index + period: SnapshotPeriod @index outstandingPrincipal: BigInt outstandingInterest: BigInt @@ -609,7 +624,7 @@ type PoolFeeSnapshot @entity { timestamp: Date! blockNumber: Int! - periodStart: Date! @index + period: SnapshotPeriod @index sumChargedAmount: BigInt #Applies to Fixed ONLY sumAccruedAmount: BigInt #Applies toChargedUpTo ONLY diff --git a/src/helpers/stateSnapshot.test.ts b/src/helpers/stateSnapshot.test.ts index 153df0ab..9676432e 100644 --- a/src/helpers/stateSnapshot.test.ts +++ b/src/helpers/stateSnapshot.test.ts @@ -2,6 +2,7 @@ import { SubstrateBlock } from '@subql/types' import { PoolService } from '../mappings/services/poolService' import { substrateStateSnapshotter } from './stateSnapshot' import { Pool, PoolSnapshot } from '../types' +import { getPeriodStart } from './timekeeperService' // eslint-disable-next-line @typescript-eslint/no-explicit-any const getByFields = store.getByFields as jest.Mock @@ -13,7 +14,8 @@ const block = { const poolId = '123456789', timestamp = new Date(), - blockNumber = 11234 + blockNumber = 11234, + periodId = getPeriodStart(timestamp).toISOString() describe('Given a populated pool,', () => { const pool = PoolService.seed(poolId) @@ -23,7 +25,7 @@ describe('Given a populated pool,', () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter(Pool, PoolSnapshot, block) + await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block) expect(store.getByFields).toHaveBeenCalledWith('Pool', [['blockchainId', '=', '0']], expect.anything()) expect(store.set).toHaveBeenNthCalledWith(1, 'Pool', poolId, expect.anything()) expect(store.set).toHaveBeenNthCalledWith(2, 'PoolSnapshot', `${poolId}-11246`, expect.anything()) @@ -33,7 +35,7 @@ describe('Given a populated pool,', () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter(Pool, PoolSnapshot, block) + await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block) expect(store.set).toHaveBeenNthCalledWith( 2, 'PoolSnapshot', @@ -46,7 +48,15 @@ describe('Given a populated pool,', () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter(Pool, PoolSnapshot, block, 'isActive', true) + await substrateStateSnapshotter( + 'periodId', + periodId, + Pool, + PoolSnapshot, + block, + 'isActive', + true + ) expect(store.getByFields).toHaveBeenNthCalledWith( 1, 'Pool', @@ -62,7 +72,16 @@ describe('Given a populated pool,', () => { set.mockReset() getByFields.mockReset() getByFields.mockReturnValue([pool]) - await substrateStateSnapshotter(Pool, PoolSnapshot, block, 'type', 'ALL', 'poolId') + await substrateStateSnapshotter( + 'periodId', + periodId, + Pool, + PoolSnapshot, + block, + 'type', + 'ALL', + 'poolId' + ) expect(store.set).toHaveBeenNthCalledWith( 2, 'PoolSnapshot', @@ -89,7 +108,7 @@ describe('Given a pool with non zero accumulators, ', () => { Object.assign(pool, accumulatedProps) - await substrateStateSnapshotter(Pool, PoolSnapshot, block) + await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block) expect(store.set).toHaveBeenNthCalledWith(1, 'Pool', poolId, expect.objectContaining(zeroedProps)) expect(store.set).toHaveBeenNthCalledWith( diff --git a/src/helpers/stateSnapshot.ts b/src/helpers/stateSnapshot.ts index 7ef37384..6bdf440d 100644 --- a/src/helpers/stateSnapshot.ts +++ b/src/helpers/stateSnapshot.ts @@ -1,5 +1,4 @@ import { EntityClass, paginatedGetter } from './paginatedGetter' -import { getPeriodStart } from './timekeeperService' import type { Entity } from '@subql/types-core' import { EthereumBlock } from '@subql/types-ethereum' import { SubstrateBlock } from '@subql/types' @@ -17,6 +16,8 @@ import { SubstrateBlock } from '@subql/types' * @returns A promise resolving when all state manipulations in the DB is completed */ async function stateSnapshotter( + relationshipField: ForeignKey, + relationshipId: string, stateModel: EntityClass, snapshotModel: EntityClass, block: { number: number; timestamp: Date }, @@ -40,7 +41,7 @@ async function stateSnapshotter( + relationshipField: ForeignKey, + relationshipId: string, stateModel: EntityClass, snapshotModel: EntityClass, block: EthereumBlock, @@ -64,10 +67,22 @@ export function evmStateSnapshotter { const formattedBlock = { number: block.number, timestamp: new Date(Number(block.timestamp) * 1000) } - return stateSnapshotter(stateModel, snapshotModel, formattedBlock, filterKey, filterValue, fkReferenceName, '1') + return stateSnapshotter( + relationshipField, + relationshipId, + stateModel, + snapshotModel, + formattedBlock, + filterKey, + filterValue, + fkReferenceName, + '1' + ) } export function substrateStateSnapshotter( + relationshipField: ForeignKey, + relationshipId: string, stateModel: EntityClass, snapshotModel: EntityClass, block: SubstrateBlock, @@ -76,7 +91,17 @@ export function substrateStateSnapshotter { const formattedBlock = { number: block.block.header.number.toNumber(), timestamp: block.timestamp } - return stateSnapshotter(stateModel, snapshotModel, formattedBlock, filterKey, filterValue, fkReferenceName, '0') + return stateSnapshotter( + relationshipField, + relationshipId, + stateModel, + snapshotModel, + formattedBlock, + filterKey, + filterValue, + fkReferenceName, + '0' + ) } type ResettableKey = `${string}ByPeriod` @@ -89,5 +114,6 @@ export interface SnapshottableEntity extends Entity { export interface SnapshottedEntityProps extends Entity { blockNumber: number timestamp: Date - periodStart: Date + periodId?: string + epochId?: string } diff --git a/src/mappings/handlers/blockHandlers.ts b/src/mappings/handlers/blockHandlers.ts index ef298db0..31b506bb 100644 --- a/src/mappings/handlers/blockHandlers.ts +++ b/src/mappings/handlers/blockHandlers.ts @@ -20,6 +20,7 @@ import { } from '../../types/models' import { AssetPositionService } from '../services/assetPositionService' import { EpochService } from '../services/epochService' +import { SnapshotPeriodService } from '../services/snapshotPeriodService' const timekeeper = TimekeeperService.init() @@ -34,14 +35,18 @@ async function _handleBlock(block: SubstrateBlock): Promise { logger.info( `It's a new period on block ${blockNumber}: ${block.timestamp.toISOString()} (specVersion: ${specVersion})` ) - const lastPeriodStart = new Date(blockPeriodStart.valueOf() - SNAPSHOT_INTERVAL_SECONDS * 1000) - const daysAgo7 = new Date(blockPeriodStart.valueOf() - 7 * 24 * 3600 * 1000) - const daysAgo30 = new Date(blockPeriodStart.valueOf() - 30 * 24 * 3600 * 1000) - const daysAgo90 = new Date(blockPeriodStart.valueOf() - 90 * 24 * 3600 * 1000) - const beginningOfMonth = new Date(blockPeriodStart.getFullYear(), blockPeriodStart.getMonth(), 1) - const quarter = Math.floor(blockPeriodStart.getMonth() / 3) - const beginningOfQuarter = new Date(blockPeriodStart.getFullYear(), quarter * 3, 1) - const beginningOfYear = new Date(blockPeriodStart.getFullYear(), 0, 1) + + const period = SnapshotPeriodService.init(blockPeriodStart) + await period.save() + + const lastPeriodStart = new Date(period.start.valueOf() - SNAPSHOT_INTERVAL_SECONDS * 1000) + const daysAgo7 = new Date(period.start.valueOf() - 7 * 24 * 3600 * 1000) + const daysAgo30 = new Date(period.start.valueOf() - 30 * 24 * 3600 * 1000) + const daysAgo90 = new Date(period.start.valueOf() - 90 * 24 * 3600 * 1000) + const beginningOfMonth = new Date(period.year, period.month, 1) + const quarter = Math.floor(period.month / 3) + const beginningOfQuarter = new Date(period.year, quarter * 3, 1) + const beginningOfYear = new Date(period.year, 0, 1) // Update Pool States const pools = await PoolService.getCfgActivePools() @@ -65,9 +70,9 @@ async function _handleBlock(block: SubstrateBlock): Promise { await tranche.computeYield('yieldYTD', beginningOfYear) await tranche.computeYield('yieldQTD', beginningOfQuarter) await tranche.computeYield('yieldMTD', beginningOfMonth) - await tranche.computeYieldAnnualized('yield7DaysAnnualized', blockPeriodStart, daysAgo7) - await tranche.computeYieldAnnualized('yield30DaysAnnualized', blockPeriodStart, daysAgo30) - await tranche.computeYieldAnnualized('yield90DaysAnnualized', blockPeriodStart, daysAgo90) + await tranche.computeYieldAnnualized('yield7DaysAnnualized', period.start, daysAgo7) + await tranche.computeYieldAnnualized('yield30DaysAnnualized', period.start, daysAgo30) + await tranche.computeYieldAnnualized('yield90DaysAnnualized', period.start, daysAgo90) await tranche.save() } // Asset operations @@ -129,12 +134,30 @@ async function _handleBlock(block: SubstrateBlock): Promise { } //Perform Snapshots and reset accumulators - await substrateStateSnapshotter(Pool, PoolSnapshot, block, 'isActive', true, 'poolId') - await substrateStateSnapshotter(Tranche, TrancheSnapshot, block, 'isActive', true, 'trancheId') - await substrateStateSnapshotter(Asset, AssetSnapshot, block, 'isActive', true, 'assetId') - await substrateStateSnapshotter(PoolFee, PoolFeeSnapshot, block, 'isActive', true, 'poolFeeId') + await substrateStateSnapshotter('periodId', period.id, Pool, PoolSnapshot, block, 'isActive', true, 'poolId') + await substrateStateSnapshotter( + 'periodId', + period.id, + Tranche, + TrancheSnapshot, + block, + 'isActive', + true, + 'trancheId' + ) + await substrateStateSnapshotter('periodId', period.id, Asset, AssetSnapshot, block, 'isActive', true, 'assetId') + await substrateStateSnapshotter( + 'periodId', + period.id, + PoolFee, + PoolFeeSnapshot, + block, + 'isActive', + true, + 'poolFeeId' + ) //Update tracking of period and continue - await (await timekeeper).update(blockPeriodStart) + await (await timekeeper).update(period.start) } } diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index f352e743..b02382a5 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -18,6 +18,7 @@ import { evmStateSnapshotter } from '../../helpers/stateSnapshot' import { Multicall3 } from '../../types/contracts/MulticallAbi' import type { Provider } from '@ethersproject/providers' import type { BigNumber } from '@ethersproject/bignumber' +import { SnapshotPeriodService } from '../services/snapshotPeriodService' const timekeeper = TimekeeperService.init() @@ -41,6 +42,9 @@ async function _handleEthBlock(block: EthereumBlock): Promise { if (newPeriod) { logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`) + const snapshotPeriod = SnapshotPeriodService.init(blockPeriodStart) + await snapshotPeriod.save() + // update pool states const poolUpdateCalls: PoolMulticall[] = [] for (const tinlakePool of tinlakePools) { @@ -129,11 +133,20 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } // Take snapshots - await evmStateSnapshotter(Pool, PoolSnapshot, block, 'isActive', true, 'poolId') + await evmStateSnapshotter( + 'periodId', + snapshotPeriod.id, + Pool, + PoolSnapshot, + block, + 'isActive', + true, + 'poolId' + ) //await evmStateSnapshotter('Asset', 'AssetSnapshot', block, 'isActive', true, 'assetId') //Update tracking of period and continue - await (await timekeeper).update(blockPeriodStart) + await (await timekeeper).update(snapshotPeriod.start) } } diff --git a/src/mappings/handlers/poolsHandlers.ts b/src/mappings/handlers/poolsHandlers.ts index 4c9a9d7e..76aca800 100644 --- a/src/mappings/handlers/poolsHandlers.ts +++ b/src/mappings/handlers/poolsHandlers.ts @@ -11,6 +11,8 @@ import { TrancheBalanceService } from '../services/trancheBalanceService' import { BlockchainService, LOCAL_CHAIN_ID } from '../services/blockchainService' import { AssetService, ONCHAIN_CASH_ASSET_ID } from '../services/assetService' import { AssetTransactionData, AssetTransactionService } from '../services/assetTransactionService' +import { substrateStateSnapshotter } from '../../helpers/stateSnapshot' +import { Pool, PoolSnapshot } from '../../types' export const handlePoolCreated = errorHandler(_handlePoolCreated) async function _handlePoolCreated(event: SubstrateEvent): Promise { @@ -284,4 +286,6 @@ async function _handleEpochExecuted(event: SubstrateEvent