feat: add storage server proxy for downloading and uploading data fro…

CHANGELOG.md

@@ -3,6 +3,7 @@
 # New features:
 
 * Add support for new format of invitation link: `c=<cid>&t=<token>&s=<serverAddress>&i=<inviterAddress>` ([#2310](https://github.com/TryQuiet/quiet/issues/2310))
+* Use server for downloading initial community metadata if v2 invitation link is detected ([#2295](https://github.com/TryQuiet/quiet/issues/2295))
 
 # Refactorings:
 

packages/backend/package.json

@@ -108,6 +108,8 @@
     "dotenv": "8.2.0",
     "events": "^3.2.0",
     "express": "^4.17.1",
+    "fetch-retry": "^6.0.0",
+    "fastq": "^1.17.1",
     "get-port": "^5.1.1",
     "go-ipfs": "npm:[email protected]",
     "http-server": "^0.12.3",

packages/backend/src/nest/connections-manager/connections-manager.module.ts

@@ -5,9 +5,10 @@ import { SocketModule } from '../socket/socket.module'
 import { StorageModule } from '../storage/storage.module'
 import { TorModule } from '../tor/tor.module'
 import { ConnectionsManagerService } from './connections-manager.service'
+import { ServerProxyServiceModule } from '../storageServerProxy/storageServerProxy.module'
 
 @Module({
-  imports: [RegistrationModule, StorageModule, TorModule, SocketModule, LocalDbModule],
+  imports: [RegistrationModule, StorageModule, TorModule, SocketModule, LocalDbModule, ServerProxyServiceModule],
   providers: [ConnectionsManagerService],
   exports: [ConnectionsManagerService],
 })

packages/backend/src/nest/connections-manager/connections-manager.service.tor.spec.ts

@@ -30,6 +30,7 @@ import waitForExpect from 'wait-for-expect'
 import { Libp2pEvents } from '../libp2p/libp2p.types'
 import { sleep } from '../common/sleep'
 import { createLibp2pAddress } from '@quiet/common'
+import { lib } from 'crypto-js'
 
 jest.setTimeout(100_000)
 
@@ -177,7 +178,6 @@ describe('Connections manager', () => {
     const spyOnDestroyHiddenService = jest.spyOn(tor, 'destroyHiddenService')
     await connectionsManagerService.init()
     const network = await connectionsManagerService.getNetwork()
-    console.log('network', network)
     expect(network.hiddenService.onionAddress.split('.')[0]).toHaveLength(56)
     expect(network.hiddenService.privateKey).toHaveLength(99)
     const peerId = await PeerId.createFromJSON(network.peerId)
@@ -216,8 +216,8 @@ describe('Connections manager', () => {
     await sleep(5000)
     // It looks LibP2P dials peers initially when it's started and
     // then IPFS service dials peers again when started, thus
-    // peersCount * 2
-    expect(spyOnDial).toHaveBeenCalledTimes(peersCount * 2)
+    // peersCount-1 * 2 because we don't dial ourself (the first peer in the list)
+    expect(spyOnDial).toHaveBeenCalledTimes((peersCount - 1) * 2)
     // Temporary fix for hanging test - websocketOverTor doesn't have abortController
     await sleep(5000)
   })

packages/backend/src/nest/connections-manager/connections-manager.service.ts

@@ -1,69 +1,68 @@
+import { peerIdFromKeys } from '@libp2p/peer-id'
 import { Inject, Injectable, OnModuleInit } from '@nestjs/common'
 import { Crypto } from '@peculiar/webcrypto'
-import { Agent } from 'https'
-import fs from 'fs'
-import path from 'path'
-import { peerIdFromKeys } from '@libp2p/peer-id'
-import { setEngine, CryptoEngine } from 'pkijs'
 import { EventEmitter } from 'events'
+import fs from 'fs'
 import getPort from 'get-port'
+import { Agent } from 'https'
+import path from 'path'
 import PeerId from 'peer-id'
+import { CryptoEngine, setEngine } from 'pkijs'
 import { getLibp2pAddressesFromCsrs, removeFilesFromDir } from '../common/utils'
 
+import { LazyModuleLoader } from '@nestjs/core'
+import { createLibp2pAddress, isPSKcodeValid, p2pAddressesToPairs } from '@quiet/common'
+import { CertFieldsTypes, getCertFieldValue, loadCertificate } from '@quiet/identity'
 import {
-  GetMessagesPayload,
   ChannelMessageIdsResponse,
-  type DeleteChannelResponse,
+  ChannelSubscribedPayload,
   ChannelsReplicatedPayload,
   Community,
-  CommunityId,
+  CommunityMetadata,
+  CommunityOwnership,
   ConnectionProcessInfo,
   CreateChannelPayload,
   CreateChannelResponse,
+  CreateNetworkPayload,
   DeleteFilesFromChannelSocketPayload,
   DownloadStatus,
   ErrorMessages,
   FileMetadata,
-  MessagesLoadedPayload,
+  GetMessagesPayload,
   InitCommunityPayload,
+  MessagesLoadedPayload,
   NetworkDataPayload,
   NetworkInfo,
   NetworkStats,
-  type SavedOwnerCertificatePayload,
   PushNotificationPayload,
-  RegisterOwnerCertificatePayload,
   RemoveDownloadStatus,
+  SaveCSRPayload,
   SendCertificatesResponse,
   SendMessagePayload,
-  ChannelSubscribedPayload,
   SocketActionTypes,
-  StorePeerListPayload,
   UploadFilePayload,
-  PeerId as PeerIdType,
-  SaveCSRPayload,
-  CommunityMetadata,
-  type PermsData,
+  type DeleteChannelResponse,
+  type SavedOwnerCertificatePayload,
   type UserProfile,
   type UserProfilesStoredEvent,
 } from '@quiet/types'
+import Logger from '../common/logger'
 import { CONFIG_OPTIONS, QUIET_DIR, SERVER_IO_PROVIDER, SOCKS_PROXY_AGENT } from '../const'
-import { ConfigOptions, GetPorts, ServerIoProviderTypes } from '../types'
-import { SocketService } from '../socket/socket.service'
-import { RegistrationService } from '../registration/registration.service'
-import { LocalDbService } from '../local-db/local-db.service'
-import { StorageService } from '../storage/storage.service'
-import { ServiceState, TorInitState } from './connections-manager.types'
 import { Libp2pService } from '../libp2p/libp2p.service'
-import { Tor } from '../tor/tor.service'
-import { LocalDBKeys } from '../local-db/local-db.types'
 import { Libp2pEvents, Libp2pNodeParams } from '../libp2p/libp2p.types'
+import { LocalDbService } from '../local-db/local-db.service'
+import { LocalDBKeys } from '../local-db/local-db.types'
+import { RegistrationService } from '../registration/registration.service'
 import { RegistrationEvents } from '../registration/registration.types'
-import { StorageEvents } from '../storage/storage.types'
-import { LazyModuleLoader } from '@nestjs/core'
-import Logger from '../common/logger'
 import { emitError } from '../socket/socket.errors'
-import { createLibp2pAddress, isPSKcodeValid } from '@quiet/common'
-import { CertFieldsTypes, createRootCA, getCertFieldValue, loadCertificate } from '@quiet/identity'
+import { SocketService } from '../socket/socket.service'
+import { StorageService } from '../storage/storage.service'
+import { StorageEvents } from '../storage/storage.types'
+import { ServerProxyService } from '../storageServerProxy/storageServerProxy.service'
+import { ServerStoredCommunityMetadata } from '../storageServerProxy/storageServerProxy.types'
+import { Tor } from '../tor/tor.service'
+import { ConfigOptions, GetPorts, ServerIoProviderTypes } from '../types'
+import { ServiceState, TorInitState } from './connections-manager.types'
 
 @Injectable()
 export class ConnectionsManagerService extends EventEmitter implements OnModuleInit {
@@ -81,6 +80,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
     @Inject(SOCKS_PROXY_AGENT) public readonly socksProxyAgent: Agent,
     private readonly socketService: SocketService,
     private readonly registrationService: RegistrationService,
+    private readonly storageServerProxyService: ServerProxyService,
     private readonly localDbService: LocalDbService,
     private readonly storageService: StorageService,
     private readonly tor: Tor,
@@ -517,7 +517,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
       agent: this.socksProxyAgent,
       localAddress: this.libp2pService.createLibp2pAddress(onionAddress, peerId.toString()),
       targetPort: this.ports.libp2pHiddenService,
-      peers: peers ?? [],
+      peers: peers ? peers.slice(1) : [],
       psk: Libp2pService.generateLibp2pPSK(community.psk).fullKey,
     }
     await this.libp2pService.createInstance(params)
@@ -628,6 +628,33 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
       await this.storageService?.saveCSR(payload)
     })
 
+    this.socketService.on(
+      SocketActionTypes.DOWNLOAD_INVITE_DATA,
+      async (payload: { cid: string; serverAddress: string }, callback: (response: CreateNetworkPayload) => void) => {
+        this.logger(`socketService - ${SocketActionTypes.DOWNLOAD_INVITE_DATA}`)
+        this.storageServerProxyService.setServerAddress(payload.serverAddress)
+        let downloadedData: ServerStoredCommunityMetadata
+        try {
+          downloadedData = await this.storageServerProxyService.downloadData(payload.cid)
+        } catch (e) {
+          this.logger.error(`Downloading community data failed`, e)
+          emitError(this.serverIoProvider.io, {
+            type: SocketActionTypes.DOWNLOAD_INVITE_DATA,
+            message: ErrorMessages.STORAGE_SERVER_CONNECTION_FAILED,
+          })
+          return
+        }
+
+        const createNetworkPayload: CreateNetworkPayload = {
+          ownership: CommunityOwnership.User,
+          peers: p2pAddressesToPairs(downloadedData.peerList),
+          psk: downloadedData.psk,
+          ownerOrbitDbIdentity: downloadedData.ownerOrbitDbIdentity,
+        }
+        callback(createNetworkPayload)
+      }
+    )
+
     // Public Channels
     this.socketService.on(
       SocketActionTypes.CREATE_CHANNEL,

packages/backend/src/nest/libp2p/libp2p.service.spec.ts

@@ -8,7 +8,7 @@ import { Libp2pEvents, Libp2pNodeParams } from './libp2p.types'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import validator from 'validator'
 import waitForExpect from 'wait-for-expect'
-import { ProcessInChunksService } from './process-in-chunks.service'
+import { DEFAULT_NUM_TRIES, ProcessInChunksService } from './process-in-chunks.service'
 
 describe('Libp2pService', () => {
   let module: TestingModule
@@ -93,10 +93,12 @@ describe('Libp2pService', () => {
     await libp2pService.createInstance(params)
     expect(libp2pService.libp2pInstance).not.toBeNull()
     // @ts-expect-error processItem is private
-    const dialPeerSpy = jest.spyOn(processInChunks, 'processItem')
+    const processItemSpy = jest.spyOn(processInChunks, 'processItem')
+    const dialSpy = jest.spyOn(libp2pService.libp2pInstance!, 'dial')
     libp2pService.emit(Libp2pEvents.DIAL_PEERS, addresses)
     await waitForExpect(async () => {
-      expect(dialPeerSpy).toBeCalledTimes(1)
+      expect(processItemSpy).toBeCalledTimes(2 * DEFAULT_NUM_TRIES)
+      expect(dialSpy).toBeCalledTimes(1)
     })
   })
 })