From 0608a513239435c3fab1837fcd64ed79df798fcb Mon Sep 17 00:00:00 2001 From: hopeyen Date: Wed, 5 Oct 2022 15:34:25 -0700 Subject: [PATCH] indexer-common,indexer-agent: auto graft resolve --- README.md | 6 ++ docs/errors.md | 10 ++ .../indexer-agent/src/__tests__/indexer.ts | 6 ++ packages/indexer-agent/src/agent.ts | 6 +- packages/indexer-agent/src/commands/start.ts | 22 +++++ packages/indexer-agent/src/indexer.ts | 59 +++++++++++- packages/indexer-common/src/errors.ts | 2 + .../src/indexer-management/allocations.ts | 8 ++ .../src/indexer-management/client.ts | 11 ++- .../resolvers/allocations.ts | 8 ++ .../src/indexer-management/subgraphs.ts | 92 ++++++++++++++++++- 11 files changed, 224 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 57e4077b4..f11612e72 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,12 @@ Indexer Infrastructure --auto-allocation-min-batch-size Minimum number of allocation transactions inside a batch for AUTO management mode [number] [default: 1] + --auto-graft-resolver-limit Maximum depth of grafting dependency to + automatically + resolve [number] [default: 0] + --ipfs-endpoint Endpoint to an ipfs node to quickly + query subgraph manifest data` [string] + [default: "https://ipfs.network.thegraph.com"] Network Subgraph --network-subgraph-deployment Network subgraph deployment [string] diff --git a/docs/errors.md b/docs/errors.md index 6a35dc32a..4578de522 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -873,3 +873,13 @@ Failed to query BlockHashFromNumber from graph node **Solution** Graph-node could not find the block hash given network and block number, check if graph-node has access to a network client that has synced to the required block. + +## IE071 + +**Summary** + +Failed to deploy subgraph deployment graft base. + +**Solution** + +Please make sure the auto graft depth resolver has correct limit, and that the graft base deployment has synced to the graft block before trying again - Set indexing rules for agent to periodically reconcile the deployment. diff --git a/packages/indexer-agent/src/__tests__/indexer.ts b/packages/indexer-agent/src/__tests__/indexer.ts index f965ed864..e1ad765dc 100644 --- a/packages/indexer-agent/src/__tests__/indexer.ts +++ b/packages/indexer-agent/src/__tests__/indexer.ts @@ -126,6 +126,7 @@ const setup = async () => { await sequelize.sync({ force: true }) const statusEndpoint = 'http://localhost:8030/graphql' + const ipfsEndpoint = 'https://ipfs.network.thegraph.com' const indexingStatusResolver = new IndexingStatusResolver({ logger: logger, statusEndpoint: 'statusEndpoint', @@ -139,6 +140,7 @@ const setup = async () => { }) const indexNodeIDs = ['node_1'] + const autoGraftResolverLimit = 1 indexerManagementClient = await createIndexerManagementClient({ models, address: toAddress(address), @@ -157,6 +159,8 @@ const setup = async () => { features: { injectDai: false, }, + ipfsEndpoint, + autoGraftResolverLimit, }) indexer = new Indexer( @@ -168,6 +172,8 @@ const setup = async () => { parseGRT('1000'), address, AllocationManagementMode.AUTO, + ipfsEndpoint, + autoGraftResolverLimit, ) } diff --git a/packages/indexer-agent/src/agent.ts b/packages/indexer-agent/src/agent.ts index fd7aebf0b..4178c85e8 100644 --- a/packages/indexer-agent/src/agent.ts +++ b/packages/indexer-agent/src/agent.ts @@ -738,7 +738,11 @@ class Agent { // Ensure the deployment is deployed to the indexer // Note: we're not waiting here, as sometimes indexing a subgraph // will block if the IPFS files cannot be retrieved - this.indexer.ensure(name, deployment) + try { + this.indexer.ensure(name, deployment) + } catch { + this.indexer.resolveGrafting(deployment, 0) + } }), ) diff --git a/packages/indexer-agent/src/commands/start.ts b/packages/indexer-agent/src/commands/start.ts index 06f55c680..0b3f9baea 100644 --- a/packages/indexer-agent/src/commands/start.ts +++ b/packages/indexer-agent/src/commands/start.ts @@ -261,6 +261,18 @@ export default { default: 100, group: 'Indexer Infrastructure', }) + .option('ipfs-endpoint', { + description: `Endpoint to an ipfs node to quickly query subgraph manifest data`, + type: 'string', + default: 'https://ipfs.network.thegraph.com', + group: 'Indexer Infrastructure', + }) + .option('auto-graft-resolver-limit', { + description: `Maximum depth of grafting dependency to automatically resolve`, + type: 'number', + default: 0, + group: 'Indexer Infrastructure', + }) .option('inject-dai', { description: 'Inject the GRT to DAI/USDC conversion rate into cost model variables', @@ -374,6 +386,12 @@ export default { ) { return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.' } + if ( + !Number.isInteger(argv['auto-graft-resolver-limit']) || + argv['auto-graft-resolver-limit'] < 0 + ) { + return 'Invalid --auto-graft-resolver-limit provided. Must be >= 0 and an integer.' + } return true }) .option('vector-node', { @@ -808,6 +826,8 @@ export default { networkMonitor, allocationManagementMode, autoAllocationMinBatchSize: argv.autoAllocationMinBatchSize, + ipfsEndpoint: argv.ipfsEndpoint, + autoGraftResolverLimit: argv.autoGraftResolverLimit, }) await createIndexerManagementServer({ @@ -826,6 +846,8 @@ export default { argv.defaultAllocationAmount, indexerAddress, allocationManagementMode, + argv.ipfsEndpoint, + argv.autoGraftResolverLimit, ) const networkSubgraphDeployment = argv.networkSubgraphDeployment ? new SubgraphDeploymentID(argv.networkSubgraphDeployment) diff --git a/packages/indexer-agent/src/indexer.ts b/packages/indexer-agent/src/indexer.ts index 110a3cc97..1e8f0e35d 100644 --- a/packages/indexer-agent/src/indexer.ts +++ b/packages/indexer-agent/src/indexer.ts @@ -28,9 +28,12 @@ import { ActionType, Allocation, AllocationDecision, + IndexingDecisionBasis, + upsertIndexingRule, } from '@graphprotocol/indexer-common' import { CombinedError } from '@urql/core' import pMap from 'p-map' +import yaml from 'yaml' const POI_DISPUTES_CONVERTERS_FROM_GRAPHQL: Record< keyof POIDisputeAttributes, @@ -84,6 +87,8 @@ export class Indexer { defaultAllocationAmount: BigNumber indexerAddress: string allocationManagementMode: AllocationManagementMode + ipfsEndpoint: string + autoGraftResolverLimit: number constructor( logger: Logger, @@ -94,12 +99,16 @@ export class Indexer { defaultAllocationAmount: BigNumber, indexerAddress: string, allocationManagementMode: AllocationManagementMode, + ipfsUrl: string, + autoGraftResolverLimit: number, ) { this.indexerManagement = indexerManagement this.statusResolver = statusResolver this.logger = logger this.indexerAddress = indexerAddress this.allocationManagementMode = allocationManagementMode + this.autoGraftResolverLimit = autoGraftResolverLimit + this.ipfsEndpoint = ipfsUrl + '/api/v0/cat?arg=' if (adminEndpoint.startsWith('https')) { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -760,6 +769,54 @@ export class Indexer { } } + // Simple fetch for subgraph manifest + async subgraphManifest(targetDeployment: SubgraphDeploymentID) { + const ipfsFile = await fetch( + this.ipfsEndpoint + targetDeployment.ipfsHash, + { + method: 'POST', + redirect: 'follow', + }, + ) + return yaml.parse(await ipfsFile.text()) + } + + // Recursive function for targetDeployment resolve grafting, add depth until reached to resolverDepth + async resolveGrafting( + targetDeployment: SubgraphDeploymentID, + depth: number, + ): Promise { + const manifest = await this.subgraphManifest(targetDeployment) + const name = `indexer-agent/${targetDeployment.ipfsHash.slice(-10)}` + + // No grafting or at root of dependency + if (!manifest.features || !manifest.features.includes('grafting')) { + if (depth) { + await this.ensure(name, targetDeployment) + } + return + } + // Default limit set to 0, disable auto-resolve of grafting dependencies + if (depth >= this.autoGraftResolverLimit) { + throw indexerError( + IndexerErrorCode.IE071, + `Grafting depth reached limit for auto resolve`, + ) + } + // If base deployment synced to required block, turn off syncing + try { + await this.resolveGrafting( + new SubgraphDeploymentID(manifest.graft.base), + depth + 1, + ) + await this.ensure(name, targetDeployment) + } catch { + throw indexerError( + IndexerErrorCode.IE071, + `Base deployment hasn't synced to the graft block, try again later`, + ) + } + } async deploy( name: string, deployment: SubgraphDeploymentID, @@ -776,7 +833,7 @@ export class Indexer { node_id: node_id, }) if (response.error) { - throw response.error + throw indexerError(IndexerErrorCode.IE026, response.error) } this.logger.info(`Successfully deployed subgraph deployment`, { name, diff --git a/packages/indexer-common/src/errors.ts b/packages/indexer-common/src/errors.ts index ce2ac65c6..b463bbece 100644 --- a/packages/indexer-common/src/errors.ts +++ b/packages/indexer-common/src/errors.ts @@ -81,6 +81,7 @@ export enum IndexerErrorCode { IE068 = 'IE068', IE069 = 'IE069', IE070 = 'IE070', + IE071 = 'IE071', } export const INDEXER_ERROR_MESSAGES: Record = { @@ -155,6 +156,7 @@ export const INDEXER_ERROR_MESSAGES: Record = { IE068: 'User-provided POI did not match reference POI from graph-node', IE069: 'Failed to query Epoch Block Oracle Subgraph', IE070: 'Failed to query BlockHashFromNumber from graph-node', + IE071: 'Failed to deploy the graft base for the target deployment', } export type IndexerErrorCause = unknown diff --git a/packages/indexer-common/src/indexer-management/allocations.ts b/packages/indexer-common/src/indexer-management/allocations.ts index da4379dbb..b228f2fd4 100644 --- a/packages/indexer-common/src/indexer-management/allocations.ts +++ b/packages/indexer-common/src/indexer-management/allocations.ts @@ -322,6 +322,14 @@ export class AllocationManager { ) } + // Ensure graft dependency is resolved + await this.subgraphManager.resolveGrafting( + logger, + this.models, + deployment, + indexNode, + 0, + ) // Ensure subgraph is deployed before allocating await this.subgraphManager.ensure( logger, diff --git a/packages/indexer-common/src/indexer-management/client.ts b/packages/indexer-common/src/indexer-management/client.ts index fae6c1440..0c9ef50bc 100644 --- a/packages/indexer-common/src/indexer-management/client.ts +++ b/packages/indexer-common/src/indexer-management/client.ts @@ -438,6 +438,8 @@ export interface IndexerManagementClientOptions { networkMonitor?: NetworkMonitor allocationManagementMode?: AllocationManagementMode autoAllocationMinBatchSize?: number + ipfsEndpoint?: string + autoGraftResolverLimit?: number } export class IndexerManagementClient extends Client { @@ -503,6 +505,8 @@ export const createIndexerManagementClient = async ( networkMonitor, allocationManagementMode, autoAllocationMinBatchSize, + ipfsEndpoint, + autoGraftResolverLimit, } = options const schema = buildSchema(print(SCHEMA_SDL)) const resolvers = { @@ -516,7 +520,12 @@ export const createIndexerManagementClient = async ( const dai: WritableEventual = mutable() - const subgraphManager = new SubgraphManager(deploymentManagementEndpoint, indexNodeIDs) + const subgraphManager = new SubgraphManager( + deploymentManagementEndpoint, + indexNodeIDs, + ipfsEndpoint, + autoGraftResolverLimit, + ) let allocationManager: AllocationManager | undefined = undefined let actionManager: ActionManager | undefined = undefined diff --git a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts index 2728308e2..60be0daa9 100644 --- a/packages/indexer-common/src/indexer-management/resolvers/allocations.ts +++ b/packages/indexer-common/src/indexer-management/resolvers/allocations.ts @@ -474,6 +474,14 @@ export default { ) } + // Ensure grafting dependencies are resolved + await subgraphManager.resolveGrafting( + logger, + models, + subgraphDeployment, + indexNode, + 0, + ) // Ensure subgraph is deployed before allocating await subgraphManager.ensure( logger, diff --git a/packages/indexer-common/src/indexer-management/subgraphs.ts b/packages/indexer-common/src/indexer-management/subgraphs.ts index 7d90386f8..6e343ac85 100644 --- a/packages/indexer-common/src/indexer-management/subgraphs.ts +++ b/packages/indexer-common/src/indexer-management/subgraphs.ts @@ -2,16 +2,31 @@ import { indexerError, IndexerErrorCode, IndexerManagementModels, + IndexingDecisionBasis, + IndexingRuleAttributes, + SubgraphIdentifierType, + upsertIndexingRule, + fetchIndexingRules, + INDEXING_RULE_GLOBAL, } from '@graphprotocol/indexer-common' import { Logger, SubgraphDeploymentID } from '@graphprotocol/common-ts' import jayson, { Client as RpcClient } from 'jayson/promise' import pTimeout from 'p-timeout' +import fetch from 'isomorphic-fetch' +import yaml from 'yaml' export class SubgraphManager { client: RpcClient indexNodeIDs: string[] + autoGraftResolverLimit: number + ipfsEndpoint?: string - constructor(endpoint: string, indexNodeIDs: string[]) { + constructor( + endpoint: string, + indexNodeIDs: string[], + ipfsUrl?: string, + autoGraftResolverLimit?: number, + ) { if (endpoint.startsWith('https')) { // eslint-disable-next-line @typescript-eslint/no-explicit-any this.client = jayson.Client.https(endpoint as any) @@ -20,6 +35,8 @@ export class SubgraphManager { this.client = jayson.Client.http(endpoint as any) } this.indexNodeIDs = indexNodeIDs + this.ipfsEndpoint = ipfsUrl + '/api/v0/cat?arg=' + this.autoGraftResolverLimit = autoGraftResolverLimit ?? 0 } async createSubgraph(logger: Logger, name: string): Promise { @@ -76,7 +93,7 @@ export class SubgraphManager { const response = await pTimeout(requestPromise, 120000) if (response.error) { - throw response.error + throw indexerError(IndexerErrorCode.IE026, response.error) } logger.info(`Successfully deployed subgraph`, { name, @@ -84,8 +101,18 @@ export class SubgraphManager { endpoints: response.result, }) - // TODO: Insert an offchain indexing rule if one matching this deployment doesn't yet exist // Will be useful for supporting deploySubgraph resolver + const indexingRules = (await fetchIndexingRules(models, false)) + .filter((rule) => rule.identifier != INDEXING_RULE_GLOBAL) + .map((rule) => new SubgraphDeploymentID(rule.identifier)) + if (!indexingRules.includes(deployment)) { + const offchainIndexingRule = { + identifier: deployment.ipfsHash, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.OFFCHAIN, + } as Partial + await upsertIndexingRule(logger, models, offchainIndexingRule) + } } catch (error) { const err = indexerError(IndexerErrorCode.IE026, error) logger.error(`Failed to deploy subgraph deployment`, { @@ -212,4 +239,63 @@ export class SubgraphManager { throw error } } + + // Simple fetch for subgraph manifest + async subgraphManifest(targetDeployment: SubgraphDeploymentID) { + const ipfsFile = await fetch(this.ipfsEndpoint + targetDeployment.ipfsHash, { + method: 'POST', + redirect: 'follow', + }) + return yaml.parse(await ipfsFile.text()) + } + + // Recursive function for targetDeployment resolve grafting, add depth until reached to resolverDepth + async resolveGrafting( + logger: Logger, + models: IndexerManagementModels, + targetDeployment: SubgraphDeploymentID, + indexNode: string | undefined, + depth: number, + ): Promise { + const manifest = await this.subgraphManifest(targetDeployment) + const name = `indexer-agent/${targetDeployment.ipfsHash.slice(-10)}` + + // No grafting or at root of dependency + if (!manifest.features || !manifest.features.includes('grafting')) { + if (depth) { + await this.ensure(logger, models, name, targetDeployment, indexNode) + } + return + } + // Default limit set to 0, disable auto-resolve of grafting dependencies + if (depth >= this.autoGraftResolverLimit) { + throw indexerError( + IndexerErrorCode.IE071, + `Grafting depth reached limit for auto resolve`, + ) + } + // If base deployment synced to required block, turn off syncing + try { + await this.resolveGrafting( + logger, + models, + new SubgraphDeploymentID(manifest.graft.base), + indexNode, + depth + 1, + ) + await this.ensure(logger, models, name, targetDeployment, indexNode) + // At this point, can safely set NEVER to graft base deployment + const offchainIndexingRule = { + identifier: manifest.graft.base, + identifierType: SubgraphIdentifierType.DEPLOYMENT, + decisionBasis: IndexingDecisionBasis.NEVER, + } as Partial + await upsertIndexingRule(logger, models, offchainIndexingRule) + } catch { + throw indexerError( + IndexerErrorCode.IE071, + `Base deployment hasn't synced to the graft block, try again later`, + ) + } + } }