From b198ce3708e7af6c27a089b7bea151d4d3b1b9e9 Mon Sep 17 00:00:00 2001 From: juanmardefago Date: Thu, 16 May 2024 02:59:03 -0300 Subject: [PATCH] feat: trying lightweight list batch update --- schema.graphql | 14 +--- src/mappings/gns.ts | 4 - src/mappings/helpers.ts | 163 ++++++++++++++++---------------------- src/mappings/l1staking.ts | 2 +- src/mappings/staking.ts | 4 +- subgraph.template.yaml | 2 +- 6 files changed, 77 insertions(+), 112 deletions(-) diff --git a/schema.graphql b/schema.graphql index ba6005f..627a563 100644 --- a/schema.graphql +++ b/schema.graphql @@ -505,6 +505,9 @@ type Indexer @entity { "Delegators to this Indexer" delegators: [DelegatedStake!]! @derivedFrom(field: "indexer") + "Delegator relation entities. For internal usage for performance reasons" + relations: [IndexerDelegatedStakeRelation!]! @derivedFrom(field: "indexer") + "CURRENT tokens delegated to the indexer" delegatedTokens: BigInt! @@ -1071,16 +1074,7 @@ type IndexerDelegatedStakeRelation @entity { id: Bytes! "Indexer entity where the delegation resides" - indexer: Indexer! - - "Delegator entity owner of said delegation stake" - delegator: Delegator! - - "DelegatedStake entity that represents the delegation" - stake: DelegatedStake! - - "Whether the delegation is active or not. Useful to avoid updating non-active delegations without deleting the entity." - active: Boolean! + indexer: Indexer } type IndexerDailyData @entity { diff --git a/src/mappings/gns.ts b/src/mappings/gns.ts index fae61fc..fc8ed7a 100644 --- a/src/mappings/gns.ts +++ b/src/mappings/gns.ts @@ -165,8 +165,6 @@ export function handleSubgraphPublished(event: SubgraphPublished): void { subgraphVersion.subgraphDeployment = subgraphDeploymentID subgraphVersion.version = versionNumber.toI32() subgraphVersion.createdAt = event.block.timestamp.toI32() - let hexHash = changetype(addQm(event.params.versionMetadata)) - let base58Hash = hexHash.toBase58() subgraphVersion.metadataHash = event.params.versionMetadata subgraphVersion.save() } @@ -637,8 +635,6 @@ export function handleSubgraphVersionUpdated(event: SubgraphVersionUpdated): voi subgraphVersion.subgraphDeployment = subgraphDeploymentID subgraphVersion.version = versionNumber.toI32() subgraphVersion.createdAt = event.block.timestamp.toI32() - let hexHash = changetype(addQm(event.params.versionMetadata)) - let base58Hash = hexHash.toBase58() subgraphVersion.metadataHash = event.params.versionMetadata subgraphVersion.save() } diff --git a/src/mappings/helpers.ts b/src/mappings/helpers.ts index 8fa8595..e4ade4c 100644 --- a/src/mappings/helpers.ts +++ b/src/mappings/helpers.ts @@ -214,10 +214,6 @@ export function createOrLoadDelegatedStake( if (delegatedStake == null) { let indexerEntity = Indexer.load(indexer)! - let relationId = compoundId( - indexer, - changetype(Bytes.fromBigInt(indexerEntity.delegatorsCount)), - ) delegatedStake = new DelegatedStake(id) delegatedStake.indexer = indexer delegatedStake.delegator = delegator.id @@ -236,15 +232,11 @@ export function createOrLoadDelegatedStake( delegatedStake.originalDelegation = BIGDECIMAL_ZERO delegatedStake.currentDelegation = BIGDECIMAL_ZERO delegatedStake.createdAt = timestamp - delegatedStake.relation = relationId + delegatedStake.relation = id delegatedStake.save() - let relation = new IndexerDelegatedStakeRelation(relationId) - - relation.indexer = indexerEntity.id - relation.stake = delegatedStake.id - relation.delegator = delegator.id - relation.active = true + let relation = new IndexerDelegatedStakeRelation(id) // minimal amount of data to avoid running out of memory + relation.indexer = indexerEntity.id // only needed for field derivation and active status relation.save() indexerEntity.delegatorsCount = indexerEntity.delegatorsCount.plus(BIGINT_ONE) @@ -623,37 +615,35 @@ export function batchUpdateDelegatorsForIndexer(indexerId: Bytes, timestamp: Big // Loading it again here to make sure we have the latest up to date data on the entity. let indexer = Indexer.load(indexerId)! // pre-calculates a lot of data for all delegators that exists for a specific indexer - // using already existing links with the indexer-delegatedStake relations - for (let i = 0; i < indexer.delegatorsCount.toI32(); i++) { - let relationId = compoundId(indexer.id, changetype(Bytes.fromBigInt(BigInt.fromString(i.toString())))) - let relation = IndexerDelegatedStakeRelation.load(relationId)! - if (relation.active) { - let delegatedStake = DelegatedStake.load(relation.stake)! - let delegator = Delegator.load(delegatedStake.delegator)! - // Only update core entities if there's a change in the exchange rate - if (delegatedStake.latestIndexerExchangeRate != indexer.delegationExchangeRate) { - let oldUnrealizedRewards = delegatedStake.unrealizedRewards - - delegatedStake.latestIndexerExchangeRate = indexer.delegationExchangeRate - delegatedStake.currentDelegation = - delegatedStake.latestIndexerExchangeRate * delegatedStake.shareAmount.toBigDecimal() - delegatedStake.unrealizedRewards = avoidNegativeRoundingError( - delegatedStake.currentDelegation - delegatedStake.originalDelegation, - ) - delegatedStake.save() - - let diffUnrealized = delegatedStake.unrealizedRewards - oldUnrealizedRewards - - delegator.totalUnrealizedRewards = avoidNegativeRoundingError( - delegator.totalUnrealizedRewards.plus(diffUnrealized), - ) - delegator.currentDelegation = delegator.currentDelegation.plus(diffUnrealized) - delegator.save() - } - - getAndUpdateDelegatedStakeDailyData(delegatedStake as DelegatedStake, timestamp) - getAndUpdateDelegatorDailyData(delegator as Delegator, timestamp) + // uses lightweight relation entity to derive the full list. hopefully it doesn't run out of memory on big deleg count indexers + let relations = indexer.relations.load() + + for (let i = 0; i < relations.length; i++) { + let delegatedStake = DelegatedStake.load(relations[i].id)! + let delegator = Delegator.load(delegatedStake.delegator)! + // Only update core entities if there's a change in the exchange rate + if (delegatedStake.latestIndexerExchangeRate != indexer.delegationExchangeRate) { + let oldUnrealizedRewards = delegatedStake.unrealizedRewards + + delegatedStake.latestIndexerExchangeRate = indexer.delegationExchangeRate + delegatedStake.currentDelegation = + delegatedStake.latestIndexerExchangeRate.times(delegatedStake.shareAmount.toBigDecimal()) + delegatedStake.unrealizedRewards = avoidNegativeRoundingError( + delegatedStake.currentDelegation.minus(delegatedStake.originalDelegation), + ) + delegatedStake.save() + + let diffUnrealized = delegatedStake.unrealizedRewards.minus(oldUnrealizedRewards) + + delegator.totalUnrealizedRewards = avoidNegativeRoundingError( + delegator.totalUnrealizedRewards.plus(diffUnrealized), + ) + delegator.currentDelegation = delegator.currentDelegation.plus(diffUnrealized) + delegator.save() } + + getAndUpdateDelegatedStakeDailyData(delegatedStake as DelegatedStake, timestamp) + getAndUpdateDelegatorDailyData(delegator as Delegator, timestamp) } } @@ -663,16 +653,13 @@ export function getAndUpdateNetworkDailyData( ): GraphNetworkDailyData { let dayNumber = timestamp.toI32() / SECONDS_PER_DAY - LAUNCH_DAY let id = compoundId(entity.id, Bytes.fromI32(dayNumber)) - let dailyData = GraphNetworkDailyData.load(id) - if (dailyData == null) { - dailyData = new GraphNetworkDailyData(id) - - dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) - dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) - dailyData.dayNumber = dayNumber - dailyData.network = entity.id - } + // not checking for previous entity since we don't want to waste a load and data will be overwritten anyways + let dailyData = new GraphNetworkDailyData(id) + dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) + dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) + dailyData.dayNumber = dayNumber + dailyData.network = entity.id dailyData.delegationRatio = entity.delegationRatio dailyData.totalTokensStaked = entity.totalTokensStaked @@ -706,18 +693,15 @@ export function getAndUpdateNetworkDailyData( export function getAndUpdateIndexerDailyData(entity: Indexer, timestamp: BigInt): IndexerDailyData { let dayNumber = timestamp.toI32() / SECONDS_PER_DAY - LAUNCH_DAY let id = compoundId(entity.id, Bytes.fromI32(dayNumber)) - let dailyData = IndexerDailyData.load(id) - if (dailyData == null) { - dailyData = new IndexerDailyData(id) - - dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) - dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) - dailyData.dayNumber = dayNumber - dailyData.indexer = entity.id - dailyData.netDailyDelegatedTokens = BIGINT_ZERO - dailyData.delegatorsCount = BIGINT_ZERO - } + // not checking for previous entity since we don't want to waste a load and data will be overwritten anyways + let dailyData = new IndexerDailyData(id) + dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) + dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) + dailyData.dayNumber = dayNumber + dailyData.indexer = entity.id + dailyData.netDailyDelegatedTokens = BIGINT_ZERO + dailyData.delegatorsCount = BIGINT_ZERO dailyData.stakedTokens = entity.stakedTokens dailyData.delegatedTokens = entity.delegatedTokens @@ -748,20 +732,17 @@ export function getAndUpdateDelegatedStakeDailyData( ): DelegatedStakeDailyData { let dayNumber = timestamp.toI32() / SECONDS_PER_DAY - LAUNCH_DAY let stakeId = compoundId(stakeEntity.id, Bytes.fromI32(dayNumber)) - let stakeDailyData = DelegatedStakeDailyData.load(stakeId) - - if (stakeDailyData == null) { - stakeDailyData = new DelegatedStakeDailyData(stakeId) - - stakeDailyData.dayStart = BigInt.fromI32( - (timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY, - ) - stakeDailyData.dayEnd = stakeDailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) - stakeDailyData.dayNumber = dayNumber - stakeDailyData.stake = stakeEntity.id - stakeDailyData.delegator = stakeEntity.delegator - stakeDailyData.indexer = stakeEntity.indexer - } + + // not checking for previous entity since we don't want to waste a load and data will be overwritten anyways + let stakeDailyData = new DelegatedStakeDailyData(stakeId) + stakeDailyData.dayStart = BigInt.fromI32( + (timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY, + ) + stakeDailyData.dayEnd = stakeDailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) + stakeDailyData.dayNumber = dayNumber + stakeDailyData.stake = stakeEntity.id + stakeDailyData.delegator = stakeEntity.delegator + stakeDailyData.indexer = stakeEntity.indexer stakeDailyData.stakedTokens = stakeEntity.stakedTokens stakeDailyData.lockedTokens = stakeEntity.lockedTokens @@ -784,16 +765,13 @@ export function getAndUpdateDelegatorDailyData( ): DelegatorDailyData { let dayNumber = timestamp.toI32() / SECONDS_PER_DAY - LAUNCH_DAY let id = compoundId(entity.id, Bytes.fromI32(dayNumber)) - let dailyData = DelegatorDailyData.load(id) - - if (dailyData == null) { - dailyData = new DelegatorDailyData(id) - - dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) - dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) - dailyData.dayNumber = dayNumber - dailyData.delegator = entity.id - } + + // not checking for previous entity since we don't want to waste a load and data will be overwritten anyways + let dailyData = new DelegatorDailyData(id) + dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) + dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) + dailyData.dayNumber = dayNumber + dailyData.delegator = entity.id dailyData.stakesCount = entity.stakesCount dailyData.activeStakesCount = entity.activeStakesCount @@ -814,16 +792,13 @@ export function getAndUpdateSubgraphDeploymentDailyData( ): SubgraphDeploymentDailyData { let dayId = timestamp.toI32() / SECONDS_PER_DAY - LAUNCH_DAY let id = compoundId(entity.id, Bytes.fromI32(dayId)) - let dailyData = SubgraphDeploymentDailyData.load(id) - - if (dailyData == null) { - dailyData = new SubgraphDeploymentDailyData(id) - dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) - dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) - dailyData.dayNumber = dayId - dailyData.subgraphDeployment = entity.id - } + // not checking for previous entity since we don't want to waste a load and data will be overwritten anyways + let dailyData = new SubgraphDeploymentDailyData(id) + dailyData.dayStart = BigInt.fromI32((timestamp.toI32() / SECONDS_PER_DAY) * SECONDS_PER_DAY) + dailyData.dayEnd = dailyData.dayStart.plus(BigInt.fromI32(SECONDS_PER_DAY)) + dailyData.dayNumber = dayId + dailyData.subgraphDeployment = entity.id dailyData.stakedTokens = entity.stakedTokens dailyData.signalledTokens = entity.signalledTokens diff --git a/src/mappings/l1staking.ts b/src/mappings/l1staking.ts index a530081..3b44ce1 100644 --- a/src/mappings/l1staking.ts +++ b/src/mappings/l1staking.ts @@ -156,7 +156,7 @@ export function handleDelegationTransferredToL2(event: DelegationTransferredToL2 // De-activate relation with indexer after batch update, so last datapoints are created properly let relation = IndexerDelegatedStakeRelation.load(delegation.relation)! - relation.active = false + relation.indexer = null relation.save() getAndUpdateNetworkDailyData(graphNetwork as GraphNetwork, event.block.timestamp) } diff --git a/src/mappings/staking.ts b/src/mappings/staking.ts index c3d8d9f..46b2927 100644 --- a/src/mappings/staking.ts +++ b/src/mappings/staking.ts @@ -271,7 +271,7 @@ export function handleStakeDelegated(event: StakeDelegated): void { // We wouldn't want to re-activate the relation in that case if (!delegatedStake.shareAmount.equals(BIGINT_ZERO)) { let relation = IndexerDelegatedStakeRelation.load(delegatedStake.relation)! - relation.active = true + relation.indexer = indexer.id relation.save() } @@ -375,7 +375,7 @@ export function handleStakeDelegatedLocked(event: StakeDelegatedLocked): void { // De-activate relation with indexer after batch update, so last datapoints are created properly if (delegatedStake.shareAmount.equals(BIGINT_ZERO)) { let relation = IndexerDelegatedStakeRelation.load(delegatedStake.relation)! - relation.active = false + relation.indexer = null relation.save() } getAndUpdateNetworkDailyData(graphNetwork as GraphNetwork, event.block.timestamp) diff --git a/subgraph.template.yaml b/subgraph.template.yaml index 3ae6d94..fb0ea29 100644 --- a/subgraph.template.yaml +++ b/subgraph.template.yaml @@ -1,4 +1,4 @@ -specVersion: 1.0.0 +specVersion: 1.1.0 description: Graph Network analytics subgraph ({{network}}) repository: https://github.com/graphprotocol/graph-network-analytics-subgraph features: