colossus s3 api

.env

@@ -186,3 +186,18 @@ SQUID_GQL_PORT=4352
 # Archive gateway host (Should not be set in local development)
 # For running a production storage-squid instance uncomment the following line (to use the subsquid hosted archive)
 # SQUID_ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-https://v2.archive.subsquid.io/network/joystream}
+
+# =====================================================================================
+## Cloud storage provider configuration
+
+# Enable or disable the cloud storage provider
+ENABLE_STORAGE_PROVIDER=true
+
+## Specify the cloud storage provider to use:
+CLOUD_STORAGE_PROVIDER_NAME=aws
+
+# AWS S3 (replace with your own configuration and credentials)
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_REGION=eu-west-1
+AWS_BUCKET_NAME=test-bucket

.gitignore

@@ -48,3 +48,6 @@ runtime-inputs/
 devops/infrastructure
 
 joystream.tar.gz
+
+# localstack for s3 backend colossus testing
+localstack/localstack_init

cli/src/Api.ts

@@ -182,7 +182,7 @@ export default class Api {
     return new Date(blockTime.toNumber())
   }
 
-  protected workingGroupApiQuery<T extends WorkingGroups>(group: T): ApiPromise['query'][typeof apiModuleByGroup[T]] {
+  protected workingGroupApiQuery<T extends WorkingGroups>(group: T): ApiPromise['query'][(typeof apiModuleByGroup)[T]] {
     const module = apiModuleByGroup[group]
     return this._api.query[module]
   }

cli/src/base/ContentDirectoryCommandBase.ts

@@ -23,10 +23,10 @@ const CATEGORIES_CONTEXTS = ['Lead', 'Curator'] as const
 const MODERATION_ACTION_CONTEXTS = ['Lead', 'Curator'] as const
 const CHANNEL_MANAGEMENT_CONTEXTS = ['Owner', 'Curator', 'Collaborator'] as const
 
-type ChannelManagementContext = typeof CHANNEL_MANAGEMENT_CONTEXTS[number]
-type ChannelCreationContext = typeof CHANNEL_CREATION_CONTEXTS[number]
-type CategoriesContext = typeof CATEGORIES_CONTEXTS[number]
-type ModerationActionContext = typeof MODERATION_ACTION_CONTEXTS[number]
+type ChannelManagementContext = (typeof CHANNEL_MANAGEMENT_CONTEXTS)[number]
+type ChannelCreationContext = (typeof CHANNEL_CREATION_CONTEXTS)[number]
+type CategoriesContext = (typeof CATEGORIES_CONTEXTS)[number]
+type ModerationActionContext = (typeof MODERATION_ACTION_CONTEXTS)[number]
 
 /**
  * Abstract base class for commands related to content directory

cli/src/base/ForumCommandBase.ts

@@ -19,7 +19,7 @@ import {
 import { AccountId } from '@polkadot/types/interfaces'
 const FORUM_MODERATION_CONTEXT = ['Leader', 'Moderator'] as const
 
-type ForumModerationContext = typeof FORUM_MODERATION_CONTEXT[number]
+type ForumModerationContext = (typeof FORUM_MODERATION_CONTEXT)[number]
 
 /**
  * Abstract base class for commands related to forum management

cli/src/commands/api/inspect.ts

@@ -22,7 +22,7 @@ const TYPES_AVAILABLE = ['query', 'consts'] as const
 
 // String literals type based on TYPES_AVAILABLE const.
 // It works as if we specified: type ApiType = 'query' | 'consts'...;
-type ApiType = typeof TYPES_AVAILABLE[number]
+type ApiType = (typeof TYPES_AVAILABLE)[number]
 
 export default class ApiInspect extends ApiCommandBase {
   static description =

cli/src/helpers/display.ts

@@ -49,7 +49,7 @@ export function displayTable(rows: { [k: string]: string | number }[], cellHoriz
     }, columnName.length)
   const columnDef = (columnName: string) => ({
     header: columnName,
-    get: (row: typeof rows[number]) => chalk.magentaBright(`${row[columnName]}`),
+    get: (row: (typeof rows)[number]) => chalk.magentaBright(`${row[columnName]}`),
     minWidth: maxLength(columnName) + cellHorizontalPadding,
   })
   const columns: Table.table.Columns<{ [k: string]: string }> = {}

docker-compose-no-bind-volumes.yml

@@ -1,5 +1,4 @@
 # Complete joystream development network
-version: '3.4'
 services:
   joystream-node:
     image: joystream/node:$JOYSTREAM_NODE_TAG
@@ -37,6 +36,9 @@ services:
       - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
       # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
       # - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
+      - AWS_BUCKET_NAME=test-bucket-1
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['yarn']
     command: [
       'start', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',
@@ -106,6 +108,9 @@ services:
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI}
+      - AWS_BUCKET_NAME=test-bucket-2
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['yarn', 'storage-node']
     command: [
       'server', '--worker=${COLOSSUS_2_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',

docker-compose.elasticsearch.yml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 services:
   # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/8.7/docker.html
   elasticsearch:

docker-compose.localstack.yml

@@ -0,0 +1,22 @@
+services:
+  localstack:
+    container_name: local_stack
+    image: localstack/localstack:3.3
+    network_mode: bridge
+    environment:
+      - SERVICES=s3 # we only need s3 bucket
+      - DEBUG=1
+      - LOCALSTACK_HOST=${LOCALSTACK_HOST}
+    ports:
+      - 4566:4566
+    volumes:
+      - localstack-data:/var/lib/localstack'
+
+networks:
+  joystream:
+    external: true
+    name: joystream_default
+
+volumes:
+  localstack-data:
+    driver: local

docker-compose.storage-squid.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   squid_db:
     container_name: squid_db

docker-compose.yml

@@ -1,5 +1,4 @@
 # Complete joystream development network
-version: '3.4'
 services:
   joystream-node:
     image: joystream/node:$JOYSTREAM_NODE_TAG
@@ -9,8 +8,8 @@ services:
       - chain-data:/data
     environment:
       - CHAIN=${CHAIN}
-    command: "--chain ${CHAIN:-dev} --alice --validator --pruning=archive --unsafe-ws-external --unsafe-rpc-external
-      --rpc-methods Safe --rpc-cors=all --log runtime --base-path /data --no-hardware-benchmarks"
+    command: '--chain ${CHAIN:-dev} --alice --validator --pruning=archive --unsafe-ws-external --unsafe-rpc-external
+      --rpc-methods Safe --rpc-cors=all --log runtime --base-path /data --no-hardware-benchmarks'
     ports:
       - 9944:9944
       - 9933:9933
@@ -37,16 +36,24 @@ services:
       - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
       - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
       - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
+      - AWS_BUCKET_NAME=test-bucket-1
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['/joystream/entrypoints/storage.sh']
-    command: [
-      'server', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data/uploads/',
-      '--sync', '--syncInterval=1',
-      '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
-      '--apiUrl=${JOYSTREAM_NODE_WS}',
-      '--logFilePath=/logs',
-      '--tempFolder=/data/temp/',
-      '--pendingFolder=/data/pending/'
-    ]
+    command:
+      [
+        'server',
+        '--worker=${COLOSSUS_1_WORKER_ID}',
+        '--port=3333',
+        '--uploads=/data/uploads/',
+        '--sync',
+        '--syncInterval=1',
+        '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
+        '--apiUrl=${JOYSTREAM_NODE_WS}',
+        '--logFilePath=/logs',
+        '--tempFolder=/data/temp/',
+        '--pendingFolder=/data/pending/',
+      ]
 
   distributor-1:
     image: node:18
@@ -71,7 +78,7 @@ services:
     environment:
       JOYSTREAM_DISTRIBUTOR__ID: distributor-1
       JOYSTREAM_DISTRIBUTOR__ENDPOINTS__STORAGE_SQUID: ${DISTRIBUTOR_STORAGE_SQUID_URL}
-      JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"${DISTRIBUTOR_1_ACCOUNT_URI}\"}]"
+      JOYSTREAM_DISTRIBUTOR__KEYS: '[{"suri":"${DISTRIBUTOR_1_ACCOUNT_URI}"}]'
       JOYSTREAM_DISTRIBUTOR__WORKER_ID: ${DISTRIBUTOR_1_WORKER_ID}
       JOYSTREAM_DISTRIBUTOR__PUBLIC_API__PORT: 3334
       JOYSTREAM_DISTRIBUTOR__OPERATOR_API__PORT: 4334
@@ -109,16 +116,24 @@ services:
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI}
+      - AWS_BUCKET_NAME=test-bucket-2
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['yarn', 'storage-node']
-    command: [
-      'server', '--worker=${COLOSSUS_2_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',
-      '--sync', '--syncInterval=1',
-      '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
-      '--apiUrl=${JOYSTREAM_NODE_WS}',
-      '--logFilePath=/logs',
-      '--tempFolder=/data/temp/',
-      '--pendingFolder=/data/pending/'
-    ]
+    command:
+      [
+        'server',
+        '--worker=${COLOSSUS_2_WORKER_ID}',
+        '--port=3333',
+        '--uploads=/data/uploads',
+        '--sync',
+        '--syncInterval=1',
+        '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
+        '--apiUrl=${JOYSTREAM_NODE_WS}',
+        '--logFilePath=/logs',
+        '--tempFolder=/data/temp/',
+        '--pendingFolder=/data/pending/',
+      ]
 
   distributor-2:
     image: node:18
@@ -143,7 +158,7 @@ services:
     environment:
       JOYSTREAM_DISTRIBUTOR__ID: distributor-2
       JOYSTREAM_DISTRIBUTOR__ENDPOINTS__STORAGE_SQUID: ${DISTRIBUTOR_STORAGE_SQUID_URL}
-      JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"${DISTRIBUTOR_2_ACCOUNT_URI}\"}]"
+      JOYSTREAM_DISTRIBUTOR__KEYS: '[{"suri":"${DISTRIBUTOR_2_ACCOUNT_URI}"}]'
       JOYSTREAM_DISTRIBUTOR__WORKER_ID: ${DISTRIBUTOR_2_WORKER_ID}
       JOYSTREAM_DISTRIBUTOR__PUBLIC_API__PORT: 3334
       JOYSTREAM_DISTRIBUTOR__OPERATOR_API__PORT: 4334
@@ -193,8 +208,8 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
       - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production
     ports:
-      - "${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}"
-      - "127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}"
+      - '${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}'
+      - '127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}'
     depends_on:
       - db
     volumes:
@@ -279,7 +294,7 @@ services:
       - PORT=${HYDRA_INDEXER_GATEWAY_PORT}
       - PGSSLMODE=disable
     ports:
-      - "${HYDRA_INDEXER_GATEWAY_PORT}:${HYDRA_INDEXER_GATEWAY_PORT}"
+      - '${HYDRA_INDEXER_GATEWAY_PORT}:${HYDRA_INDEXER_GATEWAY_PORT}'
     depends_on:
       - db
       - redis
@@ -289,7 +304,7 @@ services:
     container_name: redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      - '127.0.0.1:6379:6379'
 
   faucet:
     image: joystream/faucet:carthage
@@ -308,7 +323,7 @@ services:
       - BALANCE_CREDIT=${BALANCE_CREDIT}
       - BALANCE_LOCKED=${BALANCE_LOCKED}
     ports:
-      - "3002:3002"
+      - '3002:3002'
 
   # PostgerSQL database for Orion
   orion-db:
@@ -441,10 +456,7 @@ services:
     environment:
       DATABASE_MAX_CONNECTIONS: 5
       RUST_LOG: 'actix_web=info,actix_server=info'
-    command: [
-        '--database-url',
-        'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive',
-      ]
+    command: ['--database-url', 'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive']
     ports:
       - '127.0.0.1:${ARCHIVE_GATEWAY_PORT}:8000'
       - '[::1]:${ARCHIVE_GATEWAY_PORT}:8000'