20240425 EMBRIO Sprint 30

.envrc

@@ -1,4 +1,4 @@
-export COMPOSE_PROFILES=subql-eth
+export COMPOSE_PROFILES=subql-cfg
 export DB_USER=postgres
 export DB_PASS=postgres
 export DB_DATABASE=postgres

.github/workflows/deploy.yml

@@ -1,7 +1,5 @@
 name: SubQL Deploy
-
 # **What it does**: This action deploys to subql
-
 on:
   push:
     branches:
@@ -14,22 +12,39 @@ permissions:
   issues: write
 
 jobs:
-  subql_deploy_embrio_staging:
+  subql_deploy_staging:
     if: ${{ github.repository == 'embrio-tech/centrifuge-subql'}}
-    name: Deploy to SubQL (EMBRIO staging)
+    name: Deploy to SubQL
     strategy:
       matrix:
-        chainId: [development-embrio] # ADD ARRAY OF CHAINS TO DEPLOY
+        chainId: [demo]
     uses: ./.github/workflows/subql_deploy_workflow.yaml
     with:
       chainId: ${{ matrix.chainId }}
-      projOrg: embrio-tech
-      projImage: 'https://explorer.subquery.network/uploads/1660216119964.png'
-      deploymentType: primary
-      resetProject: true
+      projOrg: centrifuge
+      projImage: 'https://centrifuge.io/static/bfb7682cb6ed4aa422d9d2c90cd9351a/centrifuge-logomark-black.svg'
+      deploymentType: stage
+      resetProject: false
     secrets:
       accessToken: ${{ secrets.SUBQL_ACCESS_TOKEN }}
-      ethNodeApiKey: ${{ secrets.ETH_NODE_API_KEY }}
+      onfinalityApiKey: ${{ secrets.ONFINALITY_API_KEY }}
+
+  subql_deploy_staging_multichain:
+    if: ${{ github.repository == 'embrio-tech/centrifuge-subql'}}
+    name: Deploy to SubQL Multichain
+    strategy:
+      matrix:
+        chainId: [demo, centrifuge]
+    uses: ./.github/workflows/subql_multi_deploy_workflow.yaml
+    with:
+      chainId: ${{ matrix.chainId }}
+      projOrg: centrifuge
+      projImage: 'https://centrifuge.io/static/bfb7682cb6ed4aa422d9d2c90cd9351a/centrifuge-logomark-black.svg'
+      deploymentType: stage
+      resetProject: false
+    secrets:
+      accessToken: ${{ secrets.SUBQL_ACCESS_TOKEN }}
+      onfinalityApiKey: ${{ secrets.ONFINALITY_API_KEY }}
 
   subql_deploy_centrifuge_staging:
     if: ${{ github.repository == 'centrifuge/pools-subql' && github.ref_name == 'main' }}
@@ -46,7 +61,7 @@ jobs:
       resetProject: true
     secrets:
       accessToken: ${{ secrets.SUBQL_ACCESS_TOKEN }}
-      ethNodeApiKey: ${{ secrets.ETH_NODE_API_KEY }}
+      onfinalityApiKey: ${{ secrets.ONFINALITY_API_KEY }}
 
   subql_deploy_centrifuge_production:
     if: ${{ github.repository == 'centrifuge/pools-subql' && startsWith(github.ref_name, 'release-') }}
@@ -63,4 +78,21 @@ jobs:
       resetProject: false
     secrets:
       accessToken: ${{ secrets.SUBQL_ACCESS_TOKEN }}
-      ethNodeApiKey: ${{ secrets.ETH_NODE_API_KEY }}
+      onfinalityApiKey: ${{ secrets.ONFINALITY_API_KEY }}
+
+  subql_deploy_centrifuge_multichain_production:
+    if: ${{ github.repository == 'centrifuge/pools-subql' && startsWith(github.ref_name, 'release-') }}
+    name: Deploy to SubQL multichain (CENTRIFUGE mainnet)
+    strategy:
+      matrix:
+        chainId: [centrifuge] # ADD ARRAY OF CHAINS TO DEPLOY
+    uses: ./.github/workflows/subql_multi_deploy_workflow.yaml
+    with:
+      chainId: ${{ matrix.chainId }}
+      projOrg: centrifuge
+      projImage: 'https://centrifuge.io/static/bfb7682cb6ed4aa422d9d2c90cd9351a/centrifuge-logomark-black.svg'
+      deploymentType: stage
+      resetProject: false
+    secrets:
+      accessToken: ${{ secrets.SUBQL_ACCESS_TOKEN }}
+      onfinalityApiKey: ${{ secrets.ONFINALITY_API_KEY }}

.github/workflows/subql_deploy_workflow.yaml

@@ -21,22 +21,21 @@ on:
     secrets:
       accessToken:
         required: true
-      ethNodeApiKey:
+      onfinalityApiKey:
         required: true
 
 jobs:
   subql_deploy_workflow:
     runs-on: ubuntu-latest
     env:
-      SUBQL_INDEXER_VERSION: v3.6.1
-      SUBQL_ETH_INDEXER_VERSION: v3.7.0
-      SUBQL_QUERY_VERSION: v2.9.0
+      SUBQL_INDEXER_VERSION: v4.0.1
+      SUBQL_QUERY_VERSION: v2.10.0
       CHAIN_ID: ${{ inputs.chainId }}
       SUBQL_ACCESS_TOKEN: ${{ secrets.accessToken }}
       SUBQL_PROJ_ORG: ${{ inputs.projOrg }}
       SUBQL_PROJ_IMAGE: ${{ inputs.projImage }}
       SUBQL_DEPLOYMENT_TYPE: ${{ inputs.deploymentType }}
-      ETH_NODE_API_KEY: ${{ secrets.ethNodeApiKey }}
+      ONFINALITY_API_KEY: ${{ secrets.onfinalityApiKey }}
     steps:
       - name: Check out repo's default branch
         uses: actions/checkout@v3
@@ -96,5 +95,4 @@ jobs:
           --indexerVersion="$SUBQL_INDEXER_VERSION" \
           --indexerUnsafe \
           --queryVersion="$SUBQL_QUERY_VERSION" \
-          --queryAggregate
-
+          --queryAggregate

.github/workflows/subql_multi_deploy_workflow.yaml

@@ -0,0 +1,115 @@
+on:
+  workflow_call:
+    inputs:
+      chainId:
+        required: true
+        type: string
+      projOrg:
+        required: true
+        type: string
+      projImage:
+        required: false
+        type: string
+      deploymentType:
+        description: stage or primary
+        required: false
+        type: string
+        default: stage
+      resetProject:
+        default: false
+        type: boolean
+    secrets:
+      accessToken:
+        required: true
+      onfinalityApiKey:
+        required: true
+
+jobs:
+  subql_deploy_workflow:
+    runs-on: ubuntu-latest
+    env:
+      SUBQL_INDEXER_VERSION: v4.0.1
+      SUBQL_ETH_INDEXER_VERSION: v4.1.0
+      SUBQL_QUERY_VERSION: v2.10.0
+      CHAIN_ID: ${{ inputs.chainId }}
+      SUBQL_ACCESS_TOKEN: ${{ secrets.accessToken }}
+      SUBQL_PROJ_ORG: ${{ inputs.projOrg }}
+      SUBQL_PROJ_IMAGE: ${{ inputs.projImage }}
+      SUBQL_DEPLOYMENT_TYPE: ${{ inputs.deploymentType }}
+      ONFINALITY_API_KEY: ${{ secrets.onfinalityApiKey }}
+    steps:
+      - name: Check out repo's default branch
+        uses: actions/checkout@v3
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: npm
+      - name: Setup Yarn
+        run: yarn install --frozen-lockfile
+      - name: Codegen
+        run: |
+          CHAIN_ID=centrifuge yarn projectgen
+          yarn codegen
+      - name: SubQL CLI Version
+        run: yarn subql --version
+      - name: Generate new project data
+        run: 'yarn projectgen'
+      - name: Extract Chain Parameters to ENV
+        run: |
+          SUBQL_ENDPOINTS=""
+          SUBQL_DICTIONARIES=""
+          SUBQL_VERSIONS=""
+          for projectFile in project-*.yaml; do
+            echo "Reading $projectFile"
+            chainId=$(yq '.network.chainId' $projectFile)
+            endpoint=$(yq '.network.endpoint' $projectFile)
+            dictionary=$(yq '.network.dictionary' $projectFile)
+
+            if [[ -n "$SUBQL_ENDPOINTS" ]]; then SUBQL_ENDPOINTS+=","; fi
+            SUBQL_ENDPOINTS+="${chainId}:${endpoint}"
+
+            if [[ "$dictionary" != "null" ]]; then
+              if [[ -n "$SUBQL_DICTIONARIES" ]]; then SUBQL_DICTIONARIES+=","; fi
+              SUBQL_DICTIONARIES+="${chainId}:${dictionary}"
+            fi
+          done
+          echo "SUBQL_PROJ_NAME=$(yq '.name' project-centrifuge.yaml)-multichain" >> $GITHUB_ENV
+          echo "SUBQL_PROJ_DECRIPTION=$(yq '.description' project-centrifuge.yaml)" >> $GITHUB_ENV
+          echo "SUBQL_ENDPOINTS=${SUBQL_ENDPOINTS}" >> $GITHUB_ENV
+          echo "SUBQL_DICTIONARIES=${SUBQL_DICTIONARIES}" >> $GITHUB_ENV
+      - name: Publish SubQL version
+        run: |
+          echo "IPFSCID=$(node_modules/.bin/subql publish -o)" >> $GITHUB_ENV
+      - name: Delete Project
+        if: ${{ inputs.resetProject }}
+        run: |
+          npx subql project:delete-project \
+          --org="$SUBQL_PROJ_ORG" \
+          --projectName="$SUBQL_PROJ_NAME"
+        continue-on-error: true
+      - name: Create Project
+        if: ${{ inputs.resetProject }}
+        run: |
+          npx subql project:create-project \
+          --apiVersion=2 \
+          --description="$SUBQL_PROJ_DESCRIPTION" \
+          --gitRepo="https://github.com/$GITHUB_REPOSITORY" \
+          --org="$SUBQL_PROJ_ORG" \
+          --projectName="$SUBQL_PROJ_NAME" \
+          --logoURL="$SUBQL_PROJ_IMAGE"
+      - name: Deploy Multichain Version
+        run: |
+          npx subql multi-chain:deploy \
+          -d \
+          --location=./subquery-multichain.yaml \
+          --org="$SUBQL_PROJ_ORG" \
+          --projectName="$SUBQL_PROJ_NAME" \
+          --endpoint="$SUBQL_ENDPOINTS" \
+          --dict="$SUBQL_DICTIONARIES" \
+          --type=$SUBQL_DEPLOYMENT_TYPE \
+          --disableHistorical \
+          --indexerWorkers=1 \
+          --indexerUnsafe \
+          --queryAggregate
+

chains-cfg/_root.yaml

@@ -1,5 +1,4 @@
 specVersion: 1.0.0
-name: centrifuge-pools
 description: 'SubQuery API powered by EMBRIO.tech to query Centrifuge chain data for analytics insights on how Centrifuge is unlocking economic opportunity for all by connecting people to borrow and lend money transparently and cost-effectively.'
 version: 1.0.0
 runner:
@@ -45,6 +44,11 @@ dataSources:
           filter:
             module: poolSystem
             method: EpochExecuted
+        - handler: handleMetadataSet
+          kind: substrate/EventHandler
+          filter:
+            module: poolRegistry
+            method: MetadataSet
         - handler: handleInvestOrderUpdated
           kind: substrate/EventHandler
           filter:

chains-evm/base/centrifuge.yaml

@@ -1,6 +1,6 @@
 network:
   chainId: '8453' #Base Mainnet
-  endpoint: https://mainnet.base.org
+  endpoint: "https://base.api.onfinality.io/ws?apikey=${ONFINALITY_API_KEY}"
   dictionary: "https://api.subquery.network/sq/subquery/base-dictionary"
 dataSources:
   - kind: ethereum/Runtime

chains-evm/celo/centrifuge.yaml

@@ -0,0 +1,9 @@
+network:
+  chainId: '42220' #Celo Mainnet
+  endpoint: "https://celo.api.onfinality.io/ws?apikey=${ONFINALITY_API_KEY}"
+  #dictionary: ""
+dataSources:
+  - kind: ethereum/Runtime
+    startBlock: 23428180
+    options:
+      address: '0x78E9e622A57f70F1E0Ec652A4931E4e278e58142'

chains-evm/eth/centrifuge.yaml

@@ -1,7 +1,7 @@
 network:
   chainId: '1' # Ethereum Mainnet
-  endpoint: 'https://mainnet.infura.io/v3/a4ba76cd4be643618572e7467a444e3a'
-  dictionary: 'https://gx.api.subquery.network/sq/subquery/eth-dictionary'
+  endpoint: 'https://eth.api.onfinality.io/ws?apikey=${ONFINALITY_API_KEY}'
+  #dictionary: 'https://gx.api.subquery.network/sq/subquery/eth-dictionary'
 dataSources:
   - kind: ethereum/Runtime
     startBlock: 18721030

chains-evm/eth/demo.yaml

@@ -1,9 +1,8 @@
 network:
-  chainId: '5' #Ethereum Goerli
-  endpoint: "https://goerli.infura.io/v3/a4ba76cd4be643618572e7467a444e3a"
-  dictionary: "https://dict-tyk.subquery.network/query/eth-goerli"
+  chainId: '11155111' #Ethereum Sepolia
+  endpoint: "https://sepolia.infura.io/v3/ed5e0e19bcbc427cbf8f661736d44516"
 dataSources:
   - kind: ethereum/Runtime
-    startBlock: 10055835
+    startBlock: 5232630
     options:
-      address: '0x53c155d44C03CC28f892f90aA0C442850716D75F'
+      address: '0x5c8657b827a138D52a4e3f03683A28B1FaD86893'

jest.config.ts

@@ -2,6 +2,7 @@ import type { Config } from 'jest'
 
 const config: Config = {
   verbose: true,
+  workerThreads: true,
   preset: 'ts-jest',
   testEnvironment: 'node',
   setupFiles: ['./jest/globals.js'],

package.json

@@ -43,6 +43,7 @@
     "@typescript-eslint/eslint-plugin": "^6.15.0",
     "@typescript-eslint/parser": "^6.15.0",
     "abab": "^2.0.6",
+    "envsub": "^4.1.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.0.1",

project-generator.sh

@@ -10,13 +10,15 @@ EOF
 
 #CFG
 merge-yaml -i ./chains-cfg/_root.yaml ./chains-cfg/${CHAIN_ID}.yaml -o ./project-centrifuge.yaml
+envsub ./project-centrifuge.yaml
 
 #EVM Chains
 subdirs=$(find chains-evm -mindepth 1 -maxdepth 1 -type d)
 for subdir in $subdirs; do
     if [ -e "$subdir/${CHAIN_ID}.yaml" ]; then
       echo "Generating EVM config for ${subdir##*/}"
       merge-yaml -i ./$subdir/${CHAIN_ID}.yaml ./chains-evm/_root.yaml -o ./project-${subdir##*/}.yaml
+      envsub ./project-${subdir##*/}.yaml
       echo "  - project-${subdir##*/}.yaml" >> subquery-multichain.yaml
     fi
 done