indexer-common,indexer-agent: auto graft resolve

README.md

@@ -190,6 +190,12 @@ Indexer Infrastructure
   --auto-allocation-min-batch-size                 Minimum number of allocation 
                                         transactions inside a batch for AUTO 
                                         management mode    [number] [default: 1]
+  --auto-graft-resolver-limit           Maximum depth of grafting dependency to 
+                                        automatically 
+                                        resolve            [number] [default: 0]
+  --ipfs-endpoint                       Endpoint to an ipfs node to quickly 
+                                        query subgraph manifest data`   [string]
+                                        [default: "https://ipfs.network.thegraph.com"]
 
 Network Subgraph
   --network-subgraph-deployment   Network subgraph deployment           [string]

docs/errors.md

@@ -873,3 +873,13 @@ Failed to query BlockHashFromNumber from graph node
 **Solution**
 
 Graph-node could not find the block hash given network and block number, check if graph-node has access to a network client that has synced to the required block. 
+
+## IE071
+
+**Summary**
+
+Failed to deploy subgraph deployment graft base.
+
+**Solution**
+
+Please make sure the auto graft depth resolver has correct limit, and that the graft base deployment has synced to the graft block before trying again - Set indexing rules for agent to periodically reconcile the deployment.

packages/indexer-agent/src/__tests__/indexer.ts

@@ -126,6 +126,7 @@ const setup = async () => {
   await sequelize.sync({ force: true })
 
   const statusEndpoint = 'http://localhost:8030/graphql'
+  const ipfsEndpoint = 'https://ipfs.network.thegraph.com'
   const indexingStatusResolver = new IndexingStatusResolver({
     logger: logger,
     statusEndpoint: 'statusEndpoint',
@@ -139,6 +140,7 @@ const setup = async () => {
   })
 
   const indexNodeIDs = ['node_1']
+  const autoGraftResolverLimit = 1
   indexerManagementClient = await createIndexerManagementClient({
     models,
     address: toAddress(address),
@@ -157,6 +159,8 @@ const setup = async () => {
     features: {
       injectDai: false,
     },
+    ipfsEndpoint,
+    autoGraftResolverLimit,
   })
 
   indexer = new Indexer(
@@ -168,6 +172,8 @@ const setup = async () => {
     parseGRT('1000'),
     address,
     AllocationManagementMode.AUTO,
+    ipfsEndpoint,
+    autoGraftResolverLimit,
   )
 }
 

packages/indexer-agent/src/agent.ts

@@ -738,7 +738,11 @@ class Agent {
         // Ensure the deployment is deployed to the indexer
         // Note: we're not waiting here, as sometimes indexing a subgraph
         // will block if the IPFS files cannot be retrieved
-        this.indexer.ensure(name, deployment)
+        try {
+          this.indexer.ensure(name, deployment)
+        } catch {
+          this.indexer.resolveGrafting(deployment, 0)
+        }
       }),
     )
 

packages/indexer-agent/src/commands/start.ts

@@ -261,6 +261,18 @@ export default {
         default: 100,
         group: 'Indexer Infrastructure',
       })
+      .option('ipfs-endpoint', {
+        description: `Endpoint to an ipfs node to quickly query subgraph manifest data`,
+        type: 'string',
+        default: 'https://ipfs.network.thegraph.com',
+        group: 'Indexer Infrastructure',
+      })
+      .option('auto-graft-resolver-limit', {
+        description: `Maximum depth of grafting dependency to automatically resolve`,
+        type: 'number',
+        default: 0,
+        group: 'Indexer Infrastructure',
+      })
       .option('inject-dai', {
         description:
           'Inject the GRT to DAI/USDC conversion rate into cost model variables',
@@ -374,6 +386,12 @@ export default {
         ) {
           return 'Invalid --rebate-claim-max-batch-size provided. Must be > 0 and an integer.'
         }
+        if (
+          !Number.isInteger(argv['auto-graft-resolver-limit']) ||
+          argv['auto-graft-resolver-limit'] < 0
+        ) {
+          return 'Invalid --auto-graft-resolver-limit provided. Must be >= 0 and an integer.'
+        }
         return true
       })
       .option('vector-node', {
@@ -808,6 +826,8 @@ export default {
       networkMonitor,
       allocationManagementMode,
       autoAllocationMinBatchSize: argv.autoAllocationMinBatchSize,
+      ipfsEndpoint: argv.ipfsEndpoint,
+      autoGraftResolverLimit: argv.autoGraftResolverLimit,
     })
 
     await createIndexerManagementServer({
@@ -826,6 +846,8 @@ export default {
       argv.defaultAllocationAmount,
       indexerAddress,
       allocationManagementMode,
+      argv.ipfsEndpoint,
+      argv.autoGraftResolverLimit,
     )
     const networkSubgraphDeployment = argv.networkSubgraphDeployment
       ? new SubgraphDeploymentID(argv.networkSubgraphDeployment)

packages/indexer-agent/src/indexer.ts

@@ -28,9 +28,12 @@ import {
   ActionType,
   Allocation,
   AllocationDecision,
+  IndexingDecisionBasis,
+  upsertIndexingRule,
 } from '@graphprotocol/indexer-common'
 import { CombinedError } from '@urql/core'
 import pMap from 'p-map'
+import yaml from 'yaml'
 
 const POI_DISPUTES_CONVERTERS_FROM_GRAPHQL: Record<
   keyof POIDisputeAttributes,
@@ -84,6 +87,8 @@ export class Indexer {
   defaultAllocationAmount: BigNumber
   indexerAddress: string
   allocationManagementMode: AllocationManagementMode
+  ipfsEndpoint: string
+  autoGraftResolverLimit: number
 
   constructor(
     logger: Logger,
@@ -94,12 +99,16 @@ export class Indexer {
     defaultAllocationAmount: BigNumber,
     indexerAddress: string,
     allocationManagementMode: AllocationManagementMode,
+    ipfsUrl: string,
+    autoGraftResolverLimit: number,
   ) {
     this.indexerManagement = indexerManagement
     this.statusResolver = statusResolver
     this.logger = logger
     this.indexerAddress = indexerAddress
     this.allocationManagementMode = allocationManagementMode
+    this.autoGraftResolverLimit = autoGraftResolverLimit
+    this.ipfsEndpoint = ipfsUrl + '/api/v0/cat?arg='
 
     if (adminEndpoint.startsWith('https')) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -760,6 +769,54 @@ export class Indexer {
     }
   }
 
+  // Simple fetch for subgraph manifest
+  async subgraphManifest(targetDeployment: SubgraphDeploymentID) {
+    const ipfsFile = await fetch(
+      this.ipfsEndpoint + targetDeployment.ipfsHash,
+      {
+        method: 'POST',
+        redirect: 'follow',
+      },
+    )
+    return yaml.parse(await ipfsFile.text())
+  }
+
+  // Recursive function for targetDeployment resolve grafting, add depth until reached to resolverDepth
+  async resolveGrafting(
+    targetDeployment: SubgraphDeploymentID,
+    depth: number,
+  ): Promise<void> {
+    const manifest = await this.subgraphManifest(targetDeployment)
+    const name = `indexer-agent/${targetDeployment.ipfsHash.slice(-10)}`
+
+    // No grafting or at root of dependency
+    if (!manifest.features || !manifest.features.includes('grafting')) {
+      if (depth) {
+        await this.ensure(name, targetDeployment)
+      }
+      return
+    }
+    // Default limit set to 0, disable auto-resolve of grafting dependencies
+    if (depth >= this.autoGraftResolverLimit) {
+      throw indexerError(
+        IndexerErrorCode.IE071,
+        `Grafting depth reached limit for auto resolve`,
+      )
+    }
+    // If base deployment synced to required block, turn off syncing
+    try {
+      await this.resolveGrafting(
+        new SubgraphDeploymentID(manifest.graft.base),
+        depth + 1,
+      )
+      await this.ensure(name, targetDeployment)
+    } catch {
+      throw indexerError(
+        IndexerErrorCode.IE071,
+        `Base deployment hasn't synced to the graft block, try again later`,
+      )
+    }
+  }
   async deploy(
     name: string,
     deployment: SubgraphDeploymentID,
@@ -776,7 +833,7 @@ export class Indexer {
         node_id: node_id,
       })
       if (response.error) {
-        throw response.error
+        throw indexerError(IndexerErrorCode.IE026, response.error)
       }
       this.logger.info(`Successfully deployed subgraph deployment`, {
         name,

packages/indexer-common/src/errors.ts

@@ -81,6 +81,7 @@ export enum IndexerErrorCode {
   IE068 = 'IE068',
   IE069 = 'IE069',
   IE070 = 'IE070',
+  IE071 = 'IE071',
 }
 
 export const INDEXER_ERROR_MESSAGES: Record<IndexerErrorCode, string> = {
@@ -155,6 +156,7 @@ export const INDEXER_ERROR_MESSAGES: Record<IndexerErrorCode, string> = {
   IE068: 'User-provided POI did not match reference POI from graph-node',
   IE069: 'Failed to query Epoch Block Oracle Subgraph',
   IE070: 'Failed to query BlockHashFromNumber from graph-node',
+  IE071: 'Failed to deploy the graft base for the target deployment',
 }
 
 export type IndexerErrorCause = unknown

packages/indexer-common/src/indexer-management/allocations.ts

@@ -322,6 +322,14 @@ export class AllocationManager {
       )
     }
 
+    // Ensure graft dependency is resolved
+    await this.subgraphManager.resolveGrafting(
+      logger,
+      this.models,
+      deployment,
+      indexNode,
+      0,
+    )
     // Ensure subgraph is deployed before allocating
     await this.subgraphManager.ensure(
       logger,

packages/indexer-common/src/indexer-management/client.ts

@@ -438,6 +438,8 @@ export interface IndexerManagementClientOptions {
   networkMonitor?: NetworkMonitor
   allocationManagementMode?: AllocationManagementMode
   autoAllocationMinBatchSize?: number
+  ipfsEndpoint?: string
+  autoGraftResolverLimit?: number
 }
 
 export class IndexerManagementClient extends Client {
@@ -503,6 +505,8 @@ export const createIndexerManagementClient = async (
     networkMonitor,
     allocationManagementMode,
     autoAllocationMinBatchSize,
+    ipfsEndpoint,
+    autoGraftResolverLimit,
   } = options
   const schema = buildSchema(print(SCHEMA_SDL))
   const resolvers = {
@@ -516,7 +520,12 @@ export const createIndexerManagementClient = async (
 
   const dai: WritableEventual<string> = mutable()
 
-  const subgraphManager = new SubgraphManager(deploymentManagementEndpoint, indexNodeIDs)
+  const subgraphManager = new SubgraphManager(
+    deploymentManagementEndpoint,
+    indexNodeIDs,
+    ipfsEndpoint,
+    autoGraftResolverLimit,
+  )
   let allocationManager: AllocationManager | undefined = undefined
   let actionManager: ActionManager | undefined = undefined
 

packages/indexer-common/src/indexer-management/resolvers/allocations.ts

@@ -474,6 +474,14 @@ export default {
         )
       }
 
+      // Ensure grafting dependencies are resolved
+      await subgraphManager.resolveGrafting(
+        logger,
+        models,
+        subgraphDeployment,
+        indexNode,
+        0,
+      )
       // Ensure subgraph is deployed before allocating
       await subgraphManager.ensure(
         logger,