Nara update from master

.env

@@ -22,7 +22,6 @@ TYPEORM_LOGGING=error
 BLOCK_HEIGHT=0
 
 # Query node GraphQL server port
-# Remember to change it in COLOSSUS_QUERY_NODE_URL and DISTRIBUTOR_QUERY_NODE_URL as well
 GRAPHQL_SERVER_PORT=8081
 PROCESSOR_STATE_APP_PORT=8082
 PROCESSOR_HOST=processor
@@ -48,11 +47,11 @@ GRAPHQL_SERVER_HOST=localhost
 # Websocket RPC endpoint containers will use.
 JOYSTREAM_NODE_WS=ws://joystream-node:9944/
 
-# Query node which colossus will use
-COLOSSUS_QUERY_NODE_URL=http://graphql-server:8081/graphql
+# Storage Squid (graphql-server) which colossus will use
+COLOSSUS_STORAGE_SQUID_URL=http://squid-graphql-server:4352/graphql
 
-# Query node which distributor will use
-DISTRIBUTOR_QUERY_NODE_URL=http://graphql-server:8081/graphql
+# Storage Squid (graphql-server) which distributor will use
+DISTRIBUTOR_STORAGE_SQUID_URL=http://squid-graphql-server:4352/graphql
 
 # Indexer gateway used by processor. If you don't use the local indexer set this to a remote gateway
 PROCESSOR_INDEXER_GATEWAY=http://hydra-indexer-gateway:4000/graphql
@@ -154,7 +153,7 @@ [email protected]
 SQD_DEBUG=api:*
 OPENAPI_PLAYGROUND=true
 
-ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://squid-archive-gateway:8000/graphql}
+ORION_ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://squid-archive-gateway:8000/graphql}
 
 # =====================================================================================
 
@@ -168,3 +167,22 @@ TELEMETRY_ENDPOINT=http://collector:4318
 # We do not provide a default value - scripts that startup a joystream-node service
 # Should be explicit about what version to use.
 # JOYSTREAM_NODE_TAG=latest
+
+# =====================================================================================
+## Storage-Squid configuration
+
+# Db config
+SQUID_DB_HOST=squid_db
+SQUID_DB_NAME=squid
+SQUID_DB_PASS=squid
+SQUID_DB_PORT=23332
+
+# Processor service prometheus port
+SQUID_PROCESSOR_PROMETHEUS_PORT=3338
+
+# Graphql server port
+SQUID_GQL_PORT=4352
+
+# Archive gateway host (Should not be set in local development)
+# For running a production storage-squid instance uncomment the following line (to use the subsquid hosted archive)
+# SQUID_ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-https://v2.archive.subsquid.io/network/joystream}

cli/README.md

@@ -197,6 +197,7 @@ When using the CLI for the first time there are a few common steps you might wan
 - [`joystream-cli working-groups:updateRewardAccount [ADDRESS]`](#joystream-cli-working-groupsupdaterewardaccount-address)
 - [`joystream-cli working-groups:updateRoleAccount [ADDRESS]`](#joystream-cli-working-groupsupdateroleaccount-address)
 - [`joystream-cli working-groups:updateWorkerReward WORKERID NEWREWARD`](#joystream-cli-working-groupsupdateworkerreward-workerid-newreward)
+- [`joystream-cli working-groups:verifyValidator --group=membership MEMBERID`](#joystream-cli-working-groupsverifyvalidator---groupmembership-memberid)
 
 ## `joystream-cli account:create`
 
@@ -2722,4 +2723,32 @@ OPTIONS
 
 _See code: [src/commands/working-groups/updateWorkerReward.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/updateWorkerReward.ts)_
 
+## `joystream-cli working-groups:verifyValidator --group=membership MEMBERID`
+
+Verify or un-verify the membership profile bound to a validator account. Available to membership workers only.
+
+```
+USAGE
+  $ joystream-cli working-groups:verifyValidator --group=membership MEMBERID
+
+ARGUMENTS
+  MEMBERID   ID of the membership bound to the validator account.
+
+OPTIONS
+  -g, --group=membership
+      The working group context in which the command should be executed
+      membership is the only valid value for this command.
+
+  --unverify
+      Whether the profile should be un-verified.
+
+  --useMemberId=useMemberId
+      Try using the specified member id as context whenever possible
+
+  --useWorkerId=useWorkerId
+      Try using the specified worker id as context whenever possible
+```
+
+_See code: [src/commands/working-groups/verifyValidator.ts](https://github.com/Joystream/joystream/blob/master/cli/src/commands/working-groups/verifyValidator.ts)_
+
 <!-- commandsstop -->

cli/package.json

@@ -39,7 +39,7 @@
     "cross-fetch": "^3.0.6",
     "fluent-ffmpeg": "^2.1.2",
     "form-data": "^4.0.0",
-    "graphql": "^14.7.0",
+    "graphql": "^15.3.0",
     "graphql-tag": "^2.12.6",
     "inquirer": "^7.1.0",
     "inquirer-datepicker-prompt": "^0.4.2",

cli/src/commands/working-groups/verifyValidator.ts

@@ -0,0 +1,59 @@
+import { flags } from '@oclif/command'
+import WorkingGroupsCommandBase from '../../base/WorkingGroupsCommandBase'
+import { WorkingGroups } from '../../Types'
+import { VerifyValidator, RemarkMetadataAction } from '@joystream/metadata-protobuf'
+import { metadataToString } from '../../helpers/serialization'
+import Long from 'long'
+import chalk from 'chalk'
+
+export default class VerifyValidatorCommand extends WorkingGroupsCommandBase {
+  static description =
+    'Verify or un-verify the membership profile bound to a validator account. Available to membership workers only.'
+
+  static args = [
+    {
+      name: 'memberId',
+      required: true,
+      description: 'ID of the membership bound to the validator account.',
+    },
+  ]
+
+  static flags = {
+    unverify: flags.boolean({
+      default: false,
+      description: 'Whether the profile should be un-verified.',
+    }),
+
+    ...WorkingGroupsCommandBase.flags,
+  }
+
+  async run(): Promise<void> {
+    const worker = await this.getRequiredWorkerContext()
+    if (!worker || this.group === WorkingGroups.Membership) {
+      return this.error('Only membership workers can perform this command')
+    }
+
+    const { args, flags } = this.parse(VerifyValidatorCommand)
+    const memberId = Long.fromNumber(args.memberId)
+    const verifyValidator = !flags.unverify
+
+    const meta = new RemarkMetadataAction({
+      verifyValidator: new VerifyValidator({
+        memberId,
+        isVerified: verifyValidator,
+      }),
+    })
+    const message = metadataToString(RemarkMetadataAction, meta)
+
+    const keyPair = await this.getDecodedPair(worker.roleAccount)
+    const result = await this.sendAndFollowNamedTx(keyPair, 'membershipWorkingGroup', 'workerRemark', [
+      worker.workerId,
+      message,
+    ])
+
+    const block = result.status.isInBlock ? result.status.asInBlock : undefined
+    const successMessage = `The validator profile ${args.memberId} was set to${verifyValidator ? '' : ' not'} verified.`
+    const inBlockMessage = block ? `(In block ${block})` : ''
+    this.log(chalk.green(`${successMessage} ${inBlockMessage}`))
+  }
+}

devops/kubernetes/argus/index.ts

@@ -112,7 +112,7 @@ const deployment = new k8s.apps.v1.Deployment(
               workingDir: '/joystream/distributor-node',
               env: [
                 {
-                  name: 'JOYSTREAM_DISTRIBUTOR__ENDPOINTS__QUERY_NODE',
+                  name: 'JOYSTREAM_DISTRIBUTOR__ENDPOINTS__STORAGE_SQUID',
                   value: queryNodeHost,
                 },
                 {

distributor-node/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.5.1
+
+- Added parsing of Axios errors on logger level so that we never log the whole Axios client instance (which is a circular object and causes the node to crash)
+
 ## 1.5.0
 
 - Changed Elasticsearch transport to use data streams instead of regular indices. Renamed `config.logs.elastic.index` to `config.logs.elastic.indexPrefix`. Node ID from config will be automatically appended to the index name.

distributor-node/client/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-node-client",
   "description": "Joystream autogenerated distributor node api client.",
-  "version": "1.1.0",
+  "version": "2.0.0",
   "author": "Joystream contributors",
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {

distributor-node/config.yml

@@ -1,6 +1,6 @@
 id: test-node
 endpoints:
-  queryNode: http://localhost:8081/graphql
+  storageSquid: http://localhost:4352/graphql
   joystreamNodeWs: ws://localhost:9944
 directories:
   assets: ./local/data

distributor-node/docs/schema/definition-properties-endpoints-properties-storagesquid.md

@@ -0,0 +1,7 @@
+## storageSquid Type
+
+`string`
+
+## storageSquid Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")

distributor-node/docs/schema/definition-properties-endpoints.md

@@ -6,28 +6,28 @@
 
 | Property                            | Type     | Required | Nullable       | Defined by                                                                                                                                                                                    |
 | :---------------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [queryNode](#querynode)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/queryNode")             |
+| [storageSquid](#storagesquid)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-storagesquid.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/storageSquid")       |
 | [joystreamNodeWs](#joystreamnodews) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-endpoints-properties-joystreamnodews.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/joystreamNodeWs") |
 
-## queryNode
+## storageSquid
 
-Query node graphql server uri (for example: <http://localhost:8081/graphql>)
+Storage-Squid graphql server uri (for example: <http://localhost:4352/graphql>)
 
-`queryNode`
+`storageSquid`
 
 *   is required
 
 *   Type: `string`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-querynode.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/queryNode")
+*   defined in: [Distributor node configuration](definition-properties-endpoints-properties-storagesquid.md "https://joystream.org/schemas/argus/config#/properties/endpoints/properties/storageSquid")
 
-### queryNode Type
+### storageSquid Type
 
 `string`
 
-### queryNode Constraints
+### storageSquid Constraints
 
 **URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")
 

distributor-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.5.0",
+  "version": "2.0.0",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"
@@ -12,7 +12,7 @@
     "@elastic/ecs-winston-format": "^1.1.0",
     "@joystream/metadata-protobuf": "^2.8.0",
     "@joystream/opentelemetry": "1.0.0",
-    "@joystream/storage-node-client": "^3.0.0",
+    "@joystream/storage-node-client": "^4.0.0",
     "@joystream/types": "^4.0.0",
     "@nerdwallet/apollo-cache-policies": "2.10.0",
     "@oclif/command": "^1",
@@ -31,7 +31,7 @@
     "express-winston": "^4.1.0",
     "fast-safe-stringify": "^2.1.1",
     "file-type": "^16.5.1",
-    "graphql": "^14.7.0",
+    "graphql": "^15.3.0",
     "graphql-tag": "^2.12.6",
     "inquirer": "^8.1.2",
     "js-image-generator": "^1.0.3",
@@ -138,6 +138,7 @@
     "prepack": "rm -rf lib && tsc -b && oclif-dev manifest && generate:all",
     "test": "nyc --extension .ts mocha --forbid-only \"test/**/*.test.ts\"",
     "version": "generate:docs:cli && git add docs/cli/*",
+    "generate:schema:graphql": "../scripts/save-storage-squid-schema.sh src/services/networking/query-node/schema.graphql",
     "generate:types:json-schema": "yarn ts-node --transpile-only ./src/schemas/scripts/generateTypes.ts",
     "generate:types:graphql": "yarn graphql-codegen -c ./src/services/networking/query-node/codegen.yml",
     "generate:types:public-api": "yarn openapi-typescript ./src/api-spec/public.yml -o ./src/types/generated/PublicApi.ts -c ../prettierrc.js",

distributor-node/src/schemas/configSchema.ts

@@ -27,8 +27,8 @@ export const configSchema: JSONSchema4 = objectSchema({
     endpoints: objectSchema({
       description: 'Specifies external endpoints that the distributor node will connect to',
       properties: {
-        queryNode: {
-          description: 'Query node graphql server uri (for example: http://localhost:8081/graphql)',
+        storageSquid: {
+          description: 'Storage-Squid graphql server uri (for example: http://localhost:4352/graphql)',
           type: 'string',
           format: 'uri',
         },
@@ -38,7 +38,7 @@ export const configSchema: JSONSchema4 = objectSchema({
           format: 'uri',
         },
       },
-      required: ['queryNode', 'joystreamNodeWs'],
+      required: ['storageSquid', 'joystreamNodeWs'],
     }),
     directories: objectSchema({
       description: "Specifies paths where node's data will be stored",

distributor-node/src/services/logging/LoggingService.ts

@@ -1,3 +1,4 @@
+import axios from 'axios'
 import winston, { Logger, LoggerOptions } from 'winston'
 import escFormat from '@elastic/ecs-winston-format'
 import { ElasticsearchTransport } from 'winston-elasticsearch'
@@ -8,6 +9,7 @@ import stringify from 'fast-safe-stringify'
 import NodeCache from 'node-cache'
 import path from 'path'
 import 'winston-daily-rotate-file'
+import { parseAxiosError } from '../parsers/errors'
 
 const cliColors = {
   error: 'red',
@@ -51,6 +53,15 @@ const errorFormat: (opts: ErrorFormatOpts) => Format = winston.format((info, opt
   return info
 })
 
+const axiosErrorFormat = winston.format((info) => {
+  Object.entries(info).forEach(([key, value]) => {
+    if (axios.isAxiosError(value)) {
+      info[key] = parseAxiosError(value)
+    }
+  })
+  return info
+})
+
 const cliFormat = winston.format.combine(
   winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss:ms' }),
   errorFormat({ filedName: 'err' }),
@@ -91,7 +102,7 @@ export class LoggingService {
         index,
         dataStream: true,
         level: config.logs.elastic.level,
-        format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
+        format: winston.format.combine(axiosErrorFormat(), pauseFormat({ id: 'es' }), escFormat()),
         retryLimit: 10,
         flushInterval: 5000,
         // apply custom transform so that tracing data (if present) is placed in the top level of the log
@@ -144,15 +155,15 @@ export class LoggingService {
         maxSize: config.logs.file.maxSize,
         maxFiles: config.logs.file.maxFiles,
         level: config.logs.file.level,
-        format: winston.format.combine(pauseFormat({ id: 'file' }), escFormat()),
+        format: winston.format.combine(axiosErrorFormat(), pauseFormat({ id: 'file' }), escFormat()),
       })
       transports.push(fileTransport)
     }
 
     if (config.logs?.console) {
       const consoleTransport = new winston.transports.Console({
         level: config.logs.console.level,
-        format: winston.format.combine(pauseFormat({ id: 'cli' }), cliFormat),
+        format: winston.format.combine(axiosErrorFormat(), pauseFormat({ id: 'cli' }), cliFormat),
       })
       transports.push(consoleTransport)
     }