186-Store-PoolSnapshot-data-on-epoch-close-linked-to-Epoch (#223)

feat: snapshotPeriod entity introduced
feat: execute poolSnapshots after epoch execution

BREAKING CHANGE: `periodStart` deprecated on Snapshots. Substituted by `period` foreign key

schema.graphql

@@ -3,6 +3,20 @@ type Timekeeper @entity {
   lastPeriodStart: Date!
 }
 
+type SnapshotPeriod @entity {
+  id: ID!
+  start: Date! @index
+  day: Int! @index
+  weekDay: Int! @index
+  month: Int! @index
+  year: Int! @index
+
+  poolSnapshots: [PoolSnapshot] @derivedFrom(field: "period")
+  trancheSnapshots: [TrancheSnapshot] @derivedFrom(field: "period")
+  assetSnapshots: [AssetSnapshot] @derivedFrom(field: "period")
+  poolFeeSnapshots: [PoolFeeSnapshot] @derivedFrom(field: "period")
+}
+
 type Pool @entity {
   id: ID! #poolId
   # It's not possible to simply retrieve all entities, but it is supported
@@ -92,7 +106,8 @@ type PoolSnapshot @entity {
 
   timestamp: Date!
   blockNumber: Int!
-  periodStart: Date! @index
+  period: SnapshotPeriod @index
+  epoch: Epoch @index
 
   normalizedNAV: BigInt # netAssetValue, normalized to 18 decimals
 
@@ -191,7 +206,7 @@ type TrancheSnapshot @entity {
 
   timestamp: Date!
   blockNumber: Int!
-  periodStart: Date! @index
+  period: SnapshotPeriod @index
 
   tokenSupply: BigInt
   tokenPrice: BigInt
@@ -479,7 +494,7 @@ type AssetSnapshot @entity {
 
   timestamp: Date!
   blockNumber: Int!
-  periodStart: Date! @index
+  period: SnapshotPeriod @index
 
   outstandingPrincipal: BigInt
   outstandingInterest: BigInt
@@ -609,7 +624,7 @@ type PoolFeeSnapshot @entity {
 
   timestamp: Date!
   blockNumber: Int!
-  periodStart: Date! @index
+  period: SnapshotPeriod @index
 
   sumChargedAmount: BigInt #Applies to Fixed ONLY
   sumAccruedAmount: BigInt #Applies toChargedUpTo ONLY

src/helpers/stateSnapshot.test.ts

@@ -2,6 +2,7 @@ import { SubstrateBlock } from '@subql/types'
 import { PoolService } from '../mappings/services/poolService'
 import { substrateStateSnapshotter } from './stateSnapshot'
 import { Pool, PoolSnapshot } from '../types'
+import { getPeriodStart } from './timekeeperService'
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const getByFields = store.getByFields as jest.Mock
@@ -13,7 +14,8 @@ const block = {
 
 const poolId = '123456789',
   timestamp = new Date(),
-  blockNumber = 11234
+  blockNumber = 11234,
+  periodId = getPeriodStart(timestamp).toISOString()
 
 describe('Given a populated pool,', () => {
   const pool = PoolService.seed(poolId)
@@ -23,7 +25,7 @@ describe('Given a populated pool,', () => {
     set.mockReset()
     getByFields.mockReset()
     getByFields.mockReturnValue([pool])
-    await substrateStateSnapshotter(Pool, PoolSnapshot, block)
+    await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block)
     expect(store.getByFields).toHaveBeenCalledWith('Pool', [['blockchainId', '=', '0']], expect.anything())
     expect(store.set).toHaveBeenNthCalledWith(1, 'Pool', poolId, expect.anything())
     expect(store.set).toHaveBeenNthCalledWith(2, 'PoolSnapshot', `${poolId}-11246`, expect.anything())
@@ -33,7 +35,7 @@ describe('Given a populated pool,', () => {
     set.mockReset()
     getByFields.mockReset()
     getByFields.mockReturnValue([pool])
-    await substrateStateSnapshotter(Pool, PoolSnapshot, block)
+    await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block)
     expect(store.set).toHaveBeenNthCalledWith(
       2,
       'PoolSnapshot',
@@ -46,7 +48,15 @@ describe('Given a populated pool,', () => {
     set.mockReset()
     getByFields.mockReset()
     getByFields.mockReturnValue([pool])
-    await substrateStateSnapshotter<Pool, PoolSnapshot>(Pool, PoolSnapshot, block, 'isActive', true)
+    await substrateStateSnapshotter<Pool, PoolSnapshot>(
+      'periodId',
+      periodId,
+      Pool,
+      PoolSnapshot,
+      block,
+      'isActive',
+      true
+    )
     expect(store.getByFields).toHaveBeenNthCalledWith(
       1,
       'Pool',
@@ -62,7 +72,16 @@ describe('Given a populated pool,', () => {
     set.mockReset()
     getByFields.mockReset()
     getByFields.mockReturnValue([pool])
-    await substrateStateSnapshotter<Pool, PoolSnapshot>(Pool, PoolSnapshot, block, 'type', 'ALL', 'poolId')
+    await substrateStateSnapshotter<Pool, PoolSnapshot>(
+      'periodId',
+      periodId,
+      Pool,
+      PoolSnapshot,
+      block,
+      'type',
+      'ALL',
+      'poolId'
+    )
     expect(store.set).toHaveBeenNthCalledWith(
       2,
       'PoolSnapshot',
@@ -89,7 +108,7 @@ describe('Given a pool with non zero accumulators, ', () => {
 
     Object.assign(pool, accumulatedProps)
 
-    await substrateStateSnapshotter(Pool, PoolSnapshot, block)
+    await substrateStateSnapshotter('periodId', periodId, Pool, PoolSnapshot, block)
 
     expect(store.set).toHaveBeenNthCalledWith(1, 'Pool', poolId, expect.objectContaining(zeroedProps))
     expect(store.set).toHaveBeenNthCalledWith(

src/helpers/stateSnapshot.ts

@@ -1,5 +1,4 @@
 import { EntityClass, paginatedGetter } from './paginatedGetter'
-import { getPeriodStart } from './timekeeperService'
 import type { Entity } from '@subql/types-core'
 import { EthereumBlock } from '@subql/types-ethereum'
 import { SubstrateBlock } from '@subql/types'
@@ -17,6 +16,8 @@ import { SubstrateBlock } from '@subql/types'
  * @returns A promise resolving when all state manipulations in the DB is completed
  */
 async function stateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntityProps>(
+  relationshipField: ForeignKey,
+  relationshipId: string,
   stateModel: EntityClass<T>,
   snapshotModel: EntityClass<U>,
   block: { number: number; timestamp: Date },
@@ -40,7 +41,7 @@ async function stateSnapshotter<T extends SnapshottableEntity, U extends Snapsho
       id: `${id}-${blockNumber.toString()}`,
       timestamp: block.timestamp,
       blockNumber: blockNumber,
-      periodStart: getPeriodStart(block.timestamp),
+      [relationshipField]: relationshipId,
     })
     if (fkReferenceName) snapshotEntity[fkReferenceName] = stateEntity.id
 
@@ -56,6 +57,8 @@ async function stateSnapshotter<T extends SnapshottableEntity, U extends Snapsho
   return Promise.all(entitySaves)
 }
 export function evmStateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntityProps>(
+  relationshipField: ForeignKey,
+  relationshipId: string,
   stateModel: EntityClass<T>,
   snapshotModel: EntityClass<U>,
   block: EthereumBlock,
@@ -64,10 +67,22 @@ export function evmStateSnapshotter<T extends SnapshottableEntity, U extends Sna
   fkReferenceName?: ForeignKey
 ): Promise<void[]> {
   const formattedBlock = { number: block.number, timestamp: new Date(Number(block.timestamp) * 1000) }
-  return stateSnapshotter<T, U>(stateModel, snapshotModel, formattedBlock, filterKey, filterValue, fkReferenceName, '1')
+  return stateSnapshotter<T, U>(
+    relationshipField,
+    relationshipId,
+    stateModel,
+    snapshotModel,
+    formattedBlock,
+    filterKey,
+    filterValue,
+    fkReferenceName,
+    '1'
+  )
 }
 
 export function substrateStateSnapshotter<T extends SnapshottableEntity, U extends SnapshottedEntityProps>(
+  relationshipField: ForeignKey,
+  relationshipId: string,
   stateModel: EntityClass<T>,
   snapshotModel: EntityClass<U>,
   block: SubstrateBlock,
@@ -76,7 +91,17 @@ export function substrateStateSnapshotter<T extends SnapshottableEntity, U exten
   fkReferenceName?: ForeignKey
 ): Promise<void[]> {
   const formattedBlock = { number: block.block.header.number.toNumber(), timestamp: block.timestamp }
-  return stateSnapshotter<T, U>(stateModel, snapshotModel, formattedBlock, filterKey, filterValue, fkReferenceName, '0')
+  return stateSnapshotter<T, U>(
+    relationshipField,
+    relationshipId,
+    stateModel,
+    snapshotModel,
+    formattedBlock,
+    filterKey,
+    filterValue,
+    fkReferenceName,
+    '0'
+  )
 }
 
 type ResettableKey = `${string}ByPeriod`
@@ -89,5 +114,6 @@ export interface SnapshottableEntity extends Entity {
 export interface SnapshottedEntityProps extends Entity {
   blockNumber: number
   timestamp: Date
-  periodStart: Date
+  periodId?: string
+  epochId?: string
 }

src/mappings/handlers/blockHandlers.ts

@@ -20,6 +20,7 @@ import {
 } from '../../types/models'
 import { AssetPositionService } from '../services/assetPositionService'
 import { EpochService } from '../services/epochService'
+import { SnapshotPeriodService } from '../services/snapshotPeriodService'
 
 const timekeeper = TimekeeperService.init()
 
@@ -34,14 +35,18 @@ async function _handleBlock(block: SubstrateBlock): Promise<void> {
     logger.info(
       `It's a new period on block ${blockNumber}: ${block.timestamp.toISOString()} (specVersion: ${specVersion})`
     )
-    const lastPeriodStart = new Date(blockPeriodStart.valueOf() - SNAPSHOT_INTERVAL_SECONDS * 1000)
-    const daysAgo7 = new Date(blockPeriodStart.valueOf() - 7 * 24 * 3600 * 1000)
-    const daysAgo30 = new Date(blockPeriodStart.valueOf() - 30 * 24 * 3600 * 1000)
-    const daysAgo90 = new Date(blockPeriodStart.valueOf() - 90 * 24 * 3600 * 1000)
-    const beginningOfMonth = new Date(blockPeriodStart.getFullYear(), blockPeriodStart.getMonth(), 1)
-    const quarter = Math.floor(blockPeriodStart.getMonth() / 3)
-    const beginningOfQuarter = new Date(blockPeriodStart.getFullYear(), quarter * 3, 1)
-    const beginningOfYear = new Date(blockPeriodStart.getFullYear(), 0, 1)
+
+    const period = SnapshotPeriodService.init(blockPeriodStart)
+    await period.save()
+
+    const lastPeriodStart = new Date(period.start.valueOf() - SNAPSHOT_INTERVAL_SECONDS * 1000)
+    const daysAgo7 = new Date(period.start.valueOf() - 7 * 24 * 3600 * 1000)
+    const daysAgo30 = new Date(period.start.valueOf() - 30 * 24 * 3600 * 1000)
+    const daysAgo90 = new Date(period.start.valueOf() - 90 * 24 * 3600 * 1000)
+    const beginningOfMonth = new Date(period.year, period.month, 1)
+    const quarter = Math.floor(period.month / 3)
+    const beginningOfQuarter = new Date(period.year, quarter * 3, 1)
+    const beginningOfYear = new Date(period.year, 0, 1)
 
     // Update Pool States
     const pools = await PoolService.getCfgActivePools()
@@ -65,9 +70,9 @@ async function _handleBlock(block: SubstrateBlock): Promise<void> {
         await tranche.computeYield('yieldYTD', beginningOfYear)
         await tranche.computeYield('yieldQTD', beginningOfQuarter)
         await tranche.computeYield('yieldMTD', beginningOfMonth)
-        await tranche.computeYieldAnnualized('yield7DaysAnnualized', blockPeriodStart, daysAgo7)
-        await tranche.computeYieldAnnualized('yield30DaysAnnualized', blockPeriodStart, daysAgo30)
-        await tranche.computeYieldAnnualized('yield90DaysAnnualized', blockPeriodStart, daysAgo90)
+        await tranche.computeYieldAnnualized('yield7DaysAnnualized', period.start, daysAgo7)
+        await tranche.computeYieldAnnualized('yield30DaysAnnualized', period.start, daysAgo30)
+        await tranche.computeYieldAnnualized('yield90DaysAnnualized', period.start, daysAgo90)
         await tranche.save()
       }
       // Asset operations
@@ -129,12 +134,30 @@ async function _handleBlock(block: SubstrateBlock): Promise<void> {
     }
 
     //Perform Snapshots and reset accumulators
-    await substrateStateSnapshotter(Pool, PoolSnapshot, block, 'isActive', true, 'poolId')
-    await substrateStateSnapshotter(Tranche, TrancheSnapshot, block, 'isActive', true, 'trancheId')
-    await substrateStateSnapshotter(Asset, AssetSnapshot, block, 'isActive', true, 'assetId')
-    await substrateStateSnapshotter(PoolFee, PoolFeeSnapshot, block, 'isActive', true, 'poolFeeId')
+    await substrateStateSnapshotter('periodId', period.id, Pool, PoolSnapshot, block, 'isActive', true, 'poolId')
+    await substrateStateSnapshotter(
+      'periodId',
+      period.id,
+      Tranche,
+      TrancheSnapshot,
+      block,
+      'isActive',
+      true,
+      'trancheId'
+    )
+    await substrateStateSnapshotter('periodId', period.id, Asset, AssetSnapshot, block, 'isActive', true, 'assetId')
+    await substrateStateSnapshotter(
+      'periodId',
+      period.id,
+      PoolFee,
+      PoolFeeSnapshot,
+      block,
+      'isActive',
+      true,
+      'poolFeeId'
+    )
 
     //Update tracking of period and continue
-    await (await timekeeper).update(blockPeriodStart)
+    await (await timekeeper).update(period.start)
   }
 }

src/mappings/handlers/ethHandlers.ts

@@ -18,6 +18,7 @@ import { evmStateSnapshotter } from '../../helpers/stateSnapshot'
 import { Multicall3 } from '../../types/contracts/MulticallAbi'
 import type { Provider } from '@ethersproject/providers'
 import type { BigNumber } from '@ethersproject/bignumber'
+import { SnapshotPeriodService } from '../services/snapshotPeriodService'
 
 const timekeeper = TimekeeperService.init()
 
@@ -41,6 +42,9 @@ async function _handleEthBlock(block: EthereumBlock): Promise<void> {
   if (newPeriod) {
     logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`)
 
+    const snapshotPeriod = SnapshotPeriodService.init(blockPeriodStart)
+    await snapshotPeriod.save()
+
     // update pool states
     const poolUpdateCalls: PoolMulticall[] = []
     for (const tinlakePool of tinlakePools) {
@@ -129,11 +133,20 @@ async function _handleEthBlock(block: EthereumBlock): Promise<void> {
     }
 
     // Take snapshots
-    await evmStateSnapshotter<Pool, PoolSnapshot>(Pool, PoolSnapshot, block, 'isActive', true, 'poolId')
+    await evmStateSnapshotter<Pool, PoolSnapshot>(
+      'periodId',
+      snapshotPeriod.id,
+      Pool,
+      PoolSnapshot,
+      block,
+      'isActive',
+      true,
+      'poolId'
+    )
     //await evmStateSnapshotter<Asset,AssetSnapshot>('Asset', 'AssetSnapshot', block, 'isActive', true, 'assetId')
 
     //Update tracking of period and continue
-    await (await timekeeper).update(blockPeriodStart)
+    await (await timekeeper).update(snapshotPeriod.start)
   }
 }
 

src/mappings/handlers/poolsHandlers.ts

@@ -11,6 +11,8 @@ import { TrancheBalanceService } from '../services/trancheBalanceService'
 import { BlockchainService, LOCAL_CHAIN_ID } from '../services/blockchainService'
 import { AssetService, ONCHAIN_CASH_ASSET_ID } from '../services/assetService'
 import { AssetTransactionData, AssetTransactionService } from '../services/assetTransactionService'
+import { substrateStateSnapshotter } from '../../helpers/stateSnapshot'
+import { Pool, PoolSnapshot } from '../../types'
 
 export const handlePoolCreated = errorHandler(_handlePoolCreated)
 async function _handlePoolCreated(event: SubstrateEvent<PoolCreatedEvent>): Promise<void> {
@@ -284,4 +286,6 @@ async function _handleEpochExecuted(event: SubstrateEvent<EpochClosedExecutedEve
   }
 
   await Promise.all(assetTransactionSaves)
+
+  await substrateStateSnapshotter('epochId', epoch.id, Pool, PoolSnapshot, event.block, 'isActive', true, 'poolId')
 }

src/mappings/services/assetService.ts

@@ -248,7 +248,7 @@ export class AssetService extends Asset {
   public async loadSnapshot(periodStart: Date) {
     const snapshots = await AssetSnapshot.getByFields([
       ['assetId', '=', this.id],
-      ['periodStart', '=', periodStart],
+      ['periodId', '=', periodStart.toISOString()],
     ])
     if (snapshots.length !== 1) {
       logger.warn(`Unable to load snapshot for asset ${this.id} for period ${periodStart.toISOString()}`)

src/mappings/services/snapshotPeriodService.ts

@@ -0,0 +1,12 @@
+import { SnapshotPeriod } from '../../types/models/SnapshotPeriod'
+export class SnapshotPeriodService extends SnapshotPeriod {
+  static init(periodStart: Date) {
+    const id = periodStart.toISOString()
+    const day = periodStart.getUTCDate()
+    const weekday = periodStart.getUTCDay()
+    const month = periodStart.getUTCMonth()
+    const year = periodStart.getUTCFullYear()
+    logger.info(`Initialising new SnapshotPeriod with Id ${chainId}`)
+    return new this(id,periodStart,day,weekday,month,year)
+  }
+}