Merge pull request #7 from gear-foundation/indexer

Indexer

.github/workflows/build-explorer-to-k8s.yml

@@ -70,6 +70,7 @@ jobs:
       - name: Build and push image
         uses: docker/build-push-action@v5
         with:
+          context: indexer
           file: indexer/Dockerfile-explorer
           push: true
           tags: ${{ needs.prepair.outputs.image_name }}

.github/workflows/build-indexer-to-k8s.yml

@@ -70,6 +70,7 @@ jobs:
       - name: Build and push image
         uses: docker/build-push-action@v5
         with:
+          context: indexer
           file: indexer/Dockerfile-indexer
           push: true
           tags: ${{ needs.prepair.outputs.image_name }}

.gitignore

@@ -2,3 +2,4 @@
 /Cargo.lock
 .binpath
 .DS_Store
+.idea

indexer/.dockerignore

@@ -0,0 +1,4 @@
+/.git
+/node_modules
+/lib
+/*Versions.jsonl

indexer/.env.example

@@ -0,0 +1,17 @@
+DB_NAME=squid
+DB_HOST=localhost
+DB_PORT=5432
+GQL_PORT=4350
+
+# JSON-RPC node endpoints (wss or https)
+# Use private endpoints in production!
+# Set urls via secrets if deploying to Cloud: https://docs.subsquid.io/deploy-squid/env-variables/
+# OR use the RPC proxy service: https://docs.subsquid.io/deploy-squid/rpc-proxy
+RPC_ENDPOINT=wss://testnet-archive.vara.network
+MARKETPLACE_PROGRAM=0x44842bc912e6ec8eba11cc565e373c6d805600386a9fe874765515cf303215d9
+MIN_BLOCK_NUM=3276150
+RATE_LIMIT=100
+
+# Uncommenting this line enables the debug mode
+# More info at https://docs.subsquid.io/basics/logging/
+# SQD_DEBUG=*

indexer/.eslintrc.js

@@ -0,0 +1,25 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    tsconfigRootDir : __dirname, 
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+  },
+};

indexer/.gitignore

@@ -0,0 +1,10 @@
+/node_modules
+/lib
+/builds
+
+/**Versions.json
+/**Versions.jsonl
+
+# IDE files
+/.idea
+.env

indexer/.npmrc

@@ -0,0 +1 @@
+engine-strict=true

indexer/.prettierrc

@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}

indexer/Dockerfile-explorer

@@ -0,0 +1,41 @@
+FROM node:16-alpine AS node
+FROM node AS node-with-gyp
+RUN apk add g++ make python3
+FROM node-with-gyp AS builder
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+# remove if needed
+ADD assets assets
+# remove if needed
+ADD db db
+# remove if needed
+ADD schema.graphql .
+RUN npm ci
+ADD tsconfig.json .
+ADD src src
+RUN npm run build
+
+FROM node-with-gyp AS deps
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+RUN npm ci --production
+
+FROM node AS squid
+RUN npm i -g @subsquid/cli@latest
+WORKDIR /squid
+COPY --from=deps /squid/package.json .
+COPY --from=deps /squid/package-lock.json .
+COPY --from=deps /squid/node_modules node_modules
+COPY --from=builder /squid/lib lib
+# remove if no assets folder
+COPY --from=builder /squid/assets assets
+# remove if no db folder
+COPY --from=builder /squid/db db
+# remove if no schema.graphql is in the root
+COPY --from=builder /squid/schema.graphql schema.graphql
+# remove if no commands.json is in the root
+ADD commands.json .
+ENV PROCESSOR_PROMETHEUS_PORT 3000
+CMD ["npm", "run", "serve"]

indexer/Dockerfile-indexer

@@ -0,0 +1,21 @@
+FROM node:16-alpine AS node
+RUN apk add g++ make python3
+RUN npm i -g @subsquid/cli@latest
+RUN npm i -g [email protected]
+WORKDIR /squid
+ADD package.json .
+ADD package-lock.json .
+RUN npm i @types/node
+# remove if needed
+ADD assets assets
+# remove if needed
+ADD db db
+# remove if needed
+ADD schema.graphql .
+ADD tsconfig.json .
+ADD src src
+RUN npm run build
+
+ADD commands.json .
+ENV PROCESSOR_PROMETHEUS_PORT 3000
+CMD ["sqd", "process"]

indexer/README.md

@@ -0,0 +1,224 @@
+# Squid template project
+
+A starter [Squid](https://subsquid.io) project to demonstrate its structure and conventions.
+It accumulates [kusama](https://kusama.network) account transfers and serves them via GraphQL API.
+
+## Summary
+
+- [Quickstart](#quickly-running-the-sample)
+- [Public archives for Parachains](#public-archives-for-parachains)
+- [Self-hosted archive](#self-hosted-archive)
+- [Development flow](#dev-flow)
+  - [Database Schema](#1-define-database-schema)
+  - [Entity classes](#2-generate-typeorm-classes)
+  - [DB migrations](#3-generate-database-migration)
+  - [Typegen for Events, Extrinsics and Storage Calls](#4-generate-typescript-definitions-for-substrate-events-calls-and-storage)
+- [Deploy the Squid](#deploy-the-squid)
+- [Conventions](#project-conventions)
+- [Type Bundles](#types-bundle)
+
+## Prerequisites
+
+* node 16.x
+* docker
+* npm -- note that `yarn` package manager is not supported
+
+## Quickly running the sample
+
+Example commands below use [sqd](https://docs.subsquid.io/squid-cli/).
+Please [install](https://docs.subsquid.io/squid-cli/installation/) it before proceeding.
+
+```bash
+# 1. Install dependencies
+npm ci
+
+# 2. Start target Postgres database and detach
+sqd up
+
+# 3. Build the project
+sqd build
+
+# 4. Start both the squid processor and the GraphQL server
+sqd run .
+```
+A GraphiQL playground will be available at [localhost:4350/graphql](http://localhost:4350/graphql).
+
+## Public archives for Parachains
+
+Subsquid provides archive data sources [for most parachains](https://docs.subsquid.io/substrate-indexing/supported-networks/). Use `lookupArchive(<network name>, <lookup filters>)` from `@subsquid/archive-registry` to look up the archive endpoint by the network name, e.g.
+
+```typescript
+processor.setDataSource({
+  archive: lookupArchive("kusama", { release: "ArrowSquid" })
+  //...
+});
+```
+
+To make sure you're indexing the right chain one can additionally filter by the genesis block hash:
+
+```typescript
+processor.setDataSource({
+  archive: lookupArchive("kusama", { 
+    release: "ArrowSquid",
+    genesis: "0xb0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe" 
+  }),
+  //...
+});
+```
+
+If the chain is not yet supported, you can still index it using [RPC ingestion](https://docs.subsquid.io/substrate-indexing/setup/general/#set-data-source). If you take this route, use [metadata exporer](https://github.com/subsquid/squid-sdk/tree/master/substrate/substrate-metadata-explorer) with [Substrate typegen](https://docs.subsquid.io/substrate-indexing/squid-substrate-typegen/) for help with decoding.
+
+You can also fill out this [form](https://forms.gle/Vhr3exPs4HrF4Zt36) to submit a request for an Archive/Subsquid Network dataset.
+
+## Self-hosted archive
+
+Self-hosted Archives are deprecated by the ArrowSquid release. Keep an eye on updates on [Subsquid Network](https://docs.subsquid.io/subsquid-network/) and use it instead once it is released.
+
+## Dev flow
+
+### 1. Define database schema
+
+Start development by defining the schema of the target database via `schema.graphql`.
+Schema definition consists of regular graphql type declarations annotated with custom directives.
+Full description of `schema.graphql` dialect is available [here](https://docs.subsquid.io/store/postgres/schema-file/).
+
+### 2. Generate TypeORM classes
+
+Mapping developers use [TypeORM](https://typeorm.io) entities
+to interact with the target database during data processing. All necessary entity classes are
+[generated](https://docs.subsquid.io/store/postgres/schema-file/intro/) by the squid framework from `schema.graphql`. This is done by running `npx squid-typeorm-codegen`
+or (equivalently) `sqd codegen` command.
+
+### 3. Generate database migration
+
+All database changes are applied through migration files located at `db/migrations`.
+`squid-typeorm-migration(1)` tool provides several commands to drive the process.
+It is all [TypeORM](https://typeorm.io/#/migrations) under the hood.
+
+```bash
+# Connect to database, analyze its state and generate migration to match the target schema.
+# The target schema is derived from entity classes generated earlier.
+# Don't forget to compile your entity classes beforehand!
+npx squid-typeorm-migration generate
+
+# Create template file for custom database changes
+npx squid-typeorm-migration create
+
+# Apply database migrations from `db/migrations`
+npx squid-typeorm-migration apply
+
+# Revert the last performed migration
+npx squid-typeorm-migration revert         
+```
+Available `sqd` shortcuts:
+```bash
+# Build the project, remove any old migrations, then run `npx squid-typeorm-migration generate`
+sqd migration:generate
+
+# Run npx squid-typeorm-migration apply
+sqd migration:apply
+```
+
+### 4. Generate TypeScript definitions for substrate events, calls and storage 
+
+This is an optional part, but it is very advisable. 
+
+Event, call and runtime storage data come to mapping handlers as raw untyped json. 
+While it is possible to work with raw untyped json data, 
+it's extremely error-prone and the json structure may change over time due to runtime upgrades.
+
+Squid framework provides a tool for generating type-safe wrappers around events, calls and runtime storage items for
+each historical change in the spec version. See the [Substrate typegen](https://docs.subsquid.io/substrate-indexing/squid-substrate-typegen/) documentation page.
+
+## Deploy the Squid
+
+After a local run, obtain a deployment key by signing into [Subsquid Cloud](https://app.subsquid.io) and run
+
+```sh
+npx sqd auth -k YOUR_DEPLOYMENT_KEY
+```
+
+Next, inspect the Squid CLI help to deploy and manage your squid:
+
+```sh
+npx sqd squid --help
+```
+
+For more information, consult the [Deployment Guide](https://docs.subsquid.io/deploy-squid/).
+
+## Project conventions
+
+Squid tools assume a certain project layout.
+
+* All compiled js files must reside in `lib` and all TypeScript sources in `src`. 
+The layout of `lib` must reflect `src`.
+* All TypeORM classes must be exported by `src/model/index.ts` (`lib/model` module).
+* Database schema must be defined in `schema.graphql`.
+* Database migrations must reside in `db/migrations` and must be plain js files.
+* `squid-*(1)` executables consult `.env` file for a number of environment variables.
+
+See the [full desription](https://docs.subsquid.io/basics/squid-structure/) in the documentation.
+
+## Types bundle
+
+Substrate chains that have blocks with metadata versions below 14 don't provide enough 
+information to decode their data. For those chains, external [type](https://polkadot.js.org/docs/api/start/types.extend) [definitions](https://polkadot.js.org/docs/api/start/types.extend) are required.
+
+Subsquid tools include definitions for many chains, however sometimes external 
+definitions are still required.
+
+You can pass them as a special json file (types bundle) of the following structure:
+
+```json5
+{
+  "types": {
+    "AccountId": "[u8; 32]"
+  },
+  "typesAlias": {
+    "assets": {
+      "Balance": "u64"
+    }
+  },
+  "versions": [
+    {
+      "minmax": [0, 1000], // spec version range with inclusive boundaries
+      "types": {
+        "AccountId": "[u8; 16]"
+      },
+      "typesAlias": {
+        "assets": {
+          "Balance": "u32"
+        }
+      }
+    }
+  ]
+}
+```
+
+* `.types` - scale type definitions similar to [polkadot.js types](https://polkadot.js.org/docs/api/start/types.extend#extension)
+* `.typesAlias` - similar to [polkadot.js type aliases](https://polkadot.js.org/docs/api/start/types.extend#type-clashes)
+* `.versions` - per-block range overrides/patches for above fields.
+
+All fields in the type bundle are optional and applied on top of a fixed set of well-known frame types.
+
+Note, that although the structure of subsquid types bundle is very similar to the one from polkadot.js,
+those two are not fully compatible.
+
+## Differences from polkadot.js
+
+Polkadot.js provides lots of [specialized classes](https://polkadot.js.org/docs/api/start/types.basics) for various types of data. 
+Even primitives like `u32` are exposed through special classes.
+In contrast, the squid framework works only with plain js primitives and objects.
+For instance, account data is passed to the handler context as a plain byte array.  To convert it into a standard human-readable format one should explicitly use a utility lib `@subsquid/ss58`:
+
+```typescript 
+    // ...
+    from: ss58.codec('kusama').encode(rec.from),
+    to: ss58.codec('kusama').encode(rec.to),
+```
+
+## Graphql server extensions
+
+It is possible to extend `squid-graphql-server(1)` with custom
+[type-graphql](https://typegraphql.com) resolvers and to add request validation.
+For more details, consult [docs](https://docs.subsquid.io/graphql-api/).

indexer/assets/README.MD

@@ -0,0 +1,3 @@
+# Assets 
+
+`assets` is the designated folder for any additional files to be used by the squid, for example a static data file. The folder is added by default to `Dockerfile` and is kept when the squid is deployed to the Aquairum.

indexer/assets/nft.meta.txt

@@ -0,0 +1 @@
+0002000100000000000119000000011a00000000000000012000000001210000008134a80008186e66745f696f1c4e6674496e69740000100140636f6c6c656374696f6e5f6f776e657204011c4163746f724964000118636f6e666967100118436f6e666967000148696d675f6c696e6b735f616e645f646174614001605665633c28537472696e672c20496d61676544617461293e0001487065726d697373696f6e5f746f5f6d696e745c01504f7074696f6e3c5665633c4163746f7249643e3e00000410106773746418636f6d6d6f6e287072696d6974697665731c4163746f724964000004000801205b75383b2033325d000008000003200000000c000c00000503001008186e66745f696f18436f6e66696700002c01106e616d65140118537472696e6700012c6465736372697074696f6e140118537472696e6700013c636f6c6c656374696f6e5f7461677318012c5665633c537472696e673e000144636f6c6c656374696f6e5f62616e6e6572140118537472696e6700013c636f6c6c656374696f6e5f6c6f676f140118537472696e6700013c757365725f6d696e745f6c696d69741c012c4f7074696f6e3c7533323e0001406164646974696f6e616c5f6c696e6b7324015c4f7074696f6e3c4164646974696f6e616c4c696e6b733e00011c726f79616c747930010c7531360001407061796d656e745f666f725f6d696e74340110753132380001307472616e7366657261626c6538012c4f7074696f6e3c7536343e00012073656c6c61626c6538012c4f7074696f6e3c7536343e00001400000502001800000214001c04184f7074696f6e04045401200108104e6f6e6500000010536f6d6504002000000100002000000505002404184f7074696f6e04045401280108104e6f6e6500000010536f6d6504002800000100002808186e66745f696f3c4164646974696f6e616c4c696e6b73000014013065787465726e616c5f75726c2c01384f7074696f6e3c537472696e673e00012074656c656772616d2c01384f7074696f6e3c537472696e673e00011078636f6d2c01384f7074696f6e3c537472696e673e0001186d656469756d2c01384f7074696f6e3c537472696e673e00011c646973636f72642c01384f7074696f6e3c537472696e673e00002c04184f7074696f6e04045401140108104e6f6e6500000010536f6d6504001400000100003000000504003400000507003804184f7074696f6e040454013c0108104e6f6e6500000010536f6d6504003c00000100003c000005060040000002440044000004081448004808186e66745f696f24496d6167654461746100000801306c696d69745f636f706965731c012c4f7074696f6e3c7533323e00014c6175746f5f6368616e67696e675f72756c65734c01784f7074696f6e3c5665633c2854696d655365632c20416374696f6e293e3e00004c04184f7074696f6e04045401500108104e6f6e6500000010536f6d65040050000001000050000002540054000004082058005808186e66745f696f18416374696f6e000108244368616e6765496d670400140118537472696e670000001c4164644d6574610400140118537472696e67000100005c04184f7074696f6e04045401600108104e6f6e6500000010536f6d6504006000000100006000000204006408186e66745f696f244e6674416374696f6e000138205472616e73666572080108746f04011c4163746f724964000120746f6b656e5f69643c01144e66744964000000305472616e7366657246726f6d0c011066726f6d04011c4163746f724964000108746f04011c4163746f724964000120746f6b656e5f69643c01144e6674496400010030476574546f6b656e496e666f040120746f6b656e5f69643c01144e667449640002002443616e44656c657465000300104d696e740004001c417070726f7665080108746f04011c4163746f724964000120746f6b656e5f69643c01144e66744964000500385265766f6b65417070726f76616c040120746f6b656e5f69643c01144e6674496400060018457870616e640401406164646974696f6e616c5f6c696e6b734001605665633c28537472696e672c20496d61676544617461293e000700304368616e6765436f6e666967040118636f6e666967100118436f6e666967000800244368616e6765496d67080120746f6b656e5f69643c01144e66744964000120696d675f6c696e6b140118537472696e670009002c4164644d65746164617461080120746f6b656e5f69643c01144e667449640001206d65746164617461140118537472696e67000a003c4164645573657273466f724d696e7404011475736572736001305665633c4163746f7249643e000b004444656c65746555736572466f724d696e740401107573657204011c4163746f724964000c004c4c6966745265737472696374696f6e4d696e74000d0000680418526573756c74080454016c0445017c0108084f6b04006c000000000c45727204007c00000100006c08186e66745f696f204e66744576656e740001382c5472616e736665727265640c01146f776e657204011c4163746f724964000124726563697069656e7404011c4163746f724964000120746f6b656e5f69643c01144e6674496400000044546f6b656e496e666f526563656976656414012c746f6b656e5f6f776e657204011c4163746f724964000120617070726f76616c70013c4f7074696f6e3c4163746f7249643e00012073656c6c61626c65740110626f6f6c000140636f6c6c656374696f6e5f6f776e657204011c4163746f72496400011c726f79616c747930010c7531360001002443616e44656c6574650400740110626f6f6c0002002c496e697469616c697a6564080118636f6e666967100118436f6e666967000158746f74616c5f6e756d6265725f6f665f746f6b656e7338012c4f7074696f6e3c7536343e000300184d696e746564080120746f6b656e5f69643c01144e667449640001206e66745f6461746178010c4e667400040020417070726f766564080108746f04011c4163746f724964000120746f6b656e5f69643c01144e667449640005003c417070726f76616c5265766f6b6564040120746f6b656e5f69643c01144e6674496400060020457870616e6465640801406164646974696f6e616c5f6c696e6b734001605665633c28537472696e672c20496d61676544617461293e000158746f74616c5f6e756d6265725f6f665f746f6b656e7338012c4f7074696f6e3c7536343e00070034436f6e6669674368616e676564040118636f6e666967100118436f6e66696700080030496d6167654368616e676564080120746f6b656e5f69643c01144e66744964000120696d675f6c696e6b140118537472696e67000900344d657461646174614164646564080120746f6b656e5f69643c01144e667449640001206d65746164617461140118537472696e67000a00445573657273466f724d696e74416464656404011475736572736001305665633c4163746f7249643e000b004855736572466f724d696e7444656c657465640401107573657204011c4163746f724964000c004c4c6966745265737472696374696f6e4d696e74000d00007004184f7074696f6e04045401040108104e6f6e6500000010536f6d6504000400000100007400000500007808186e66745f696f0c4e667400001801146f776e657204011c4163746f7249640001106e616d65140118537472696e6700012c6465736372697074696f6e140118537472696e670001206d6574616461746118012c5665633c537472696e673e0001246d656469615f75726c140118537472696e670001246d696e745f74696d653c010c75363400007c08186e66745f696f204e66744572726f7200000400140118537472696e6700008008186e66745f696f2853746174655175657279000110104e616d650000002c4465736372697074696f6e00010018436f6e6669670002000c416c6c000300008408186e66745f696f2853746174655265706c79000110104e616d650400140118537472696e670000002c4465736372697074696f6e0400140118537472696e6700010018436f6e6669670400100118436f6e6669670002000c416c6c04008801204e66745374617465000300008808186e66745f696f204e667453746174650000240118746f6b656e738c01445665633c284e667449642c204e6674293e0001186f776e6572739401685665633c284163746f7249642c205665633c4e667449643e293e00013c746f6b656e5f617070726f76616c73a001545665633c284e667449642c204163746f724964293e000118636f6e666967100118436f6e6669670001146e6f6e63653c01144e66744964000148696d675f6c696e6b735f616e645f646174614001605665633c28537472696e672c20496d61676544617461293e000140636f6c6c656374696f6e5f6f776e657204011c4163746f724964000158746f74616c5f6e756d6265725f6f665f746f6b656e7338012c4f7074696f6e3c7536343e0001487065726d697373696f6e5f746f5f6d696e745c01504f7074696f6e3c5665633c4163746f7249643e3e00008c000002900090000004083c78009400000298009800000408049c009c0000023c00a0000002a400a4000004083c0400