Chore/2312

.github/workflows/utils-tests.yml

@@ -0,0 +1,29 @@
+name: Common package tests
+
+on:
+  pull_request:
+    paths:
+      - packages/common/**
+
+jobs:
+  utils-tests:
+    timeout-minutes: 25
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, macos-latest, windows-2019]
+
+    steps:
+      - name: "Print OS"
+        run: echo ${{ matrix.os }}
+
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: "Setup environment"
+        uses: ./.github/actions/setup-env
+        with:
+          bootstrap-packages: "@quiet/eslint-config,@quiet/logger,@quiet/types,@quiet/common"
+
+      - name: "Unit tests"
+        run: lerna run test --scope @quiet/common --stream

CHANGELOG.md

@@ -2,9 +2,12 @@
 
 # New features:
 
+* Add support for new format of invitation link: `c=<cid>&t=<token>&s=<serverAddress>&i=<inviterAddress>` ([#2310](https://github.com/TryQuiet/quiet/issues/2310))
+* Use server for downloading initial community metadata if v2 invitation link is detected ([#2295](https://github.com/TryQuiet/quiet/issues/2295))
+
 # Refactorings:
 
-# Fixes:
+# Fixes
 
 # Chores
 

package.json

@@ -8,6 +8,8 @@
     "start:desktop": "lerna run --scope @quiet/desktop start",
     "lint:all": "lerna run lint",
     "distAndRunE2ETests:mac:local": "lerna run --scope @quiet/desktop distMac:local && lerna run --scope e2e-tests test:localBinary --",
+    "e2e:linux:build": "lerna run --scope @quiet/backend webpack:prod && lerna run --scope @quiet/desktop distUbuntu && lerna run --scope e2e-tests linux:copy",
+    "e2e:linux:run": "lerna run --scope e2e-tests test --",
     "prepare": "husky",
     "lint-staged": "lerna run lint-staged"
   },

packages/backend/jestSetup.js

@@ -1,11 +1,14 @@
-import { setEngine, CryptoEngine } from'pkijs'
+import { setEngine, CryptoEngine } from 'pkijs'
 import { Crypto } from '@peculiar/webcrypto'
 
-const crypto = new Crypto();
-global.crypto = crypto;
+const crypto = new Crypto()
+global.crypto = crypto
 
-setEngine('newEngine', new CryptoEngine({
+setEngine(
+  'newEngine',
+  new CryptoEngine({
     name: 'newEngine',
     // @ts-ignore
     crypto: crypto,
-  }))
+  })
+)

packages/backend/package.json

@@ -108,6 +108,7 @@
     "dotenv": "8.2.0",
     "events": "^3.2.0",
     "express": "^4.17.1",
+    "fetch-retry": "^6.0.0",
     "get-port": "^5.1.1",
     "go-ipfs": "npm:[email protected]",
     "http-server": "^0.12.3",

packages/backend/src/nest/connections-manager/connections-manager.module.ts

@@ -5,9 +5,10 @@ import { SocketModule } from '../socket/socket.module'
 import { StorageModule } from '../storage/storage.module'
 import { TorModule } from '../tor/tor.module'
 import { ConnectionsManagerService } from './connections-manager.service'
+import { ServerProxyServiceModule } from '../storageServerProxy/storageServerProxy.module'
 
 @Module({
-  imports: [RegistrationModule, StorageModule, TorModule, SocketModule, LocalDbModule],
+  imports: [RegistrationModule, StorageModule, TorModule, SocketModule, LocalDbModule, ServerProxyServiceModule],
   providers: [ConnectionsManagerService],
   exports: [ConnectionsManagerService],
 })

packages/backend/src/nest/connections-manager/connections-manager.service.tor.spec.ts

@@ -5,8 +5,7 @@ import { CustomEvent } from '@libp2p/interfaces/events'
 import { jest, beforeEach, describe, it, expect, afterEach } from '@jest/globals'
 import { communities, getFactory, identity, prepareStore, Store } from '@quiet/state-manager'
 import { createPeerId, createTmpDir, libp2pInstanceParams, removeFilesFromDir, tmpQuietDirPath } from '../common/utils'
-
-import { NetworkStats, type Community, type Identity, type InitCommunityPayload } from '@quiet/types'
+import { NetworkStats, type Community, type Identity } from '@quiet/types'
 import { LazyModuleLoader } from '@nestjs/core'
 import { TestingModule, Test } from '@nestjs/testing'
 import { FactoryGirl } from 'factory-girl'
@@ -113,7 +112,6 @@ beforeEach(async () => {
 })
 
 afterEach(async () => {
-  await libp2pService?.libp2pInstance?.stop()
   if (connectionsManagerService) {
     await connectionsManagerService.closeAllServices()
   }
@@ -122,6 +120,10 @@ afterEach(async () => {
 
 describe('Connections manager', () => {
   it('saves peer stats when peer has been disconnected', async () => {
+    // @ts-expect-error
+    libp2pService.processInChunksService.init = jest.fn()
+    // @ts-expect-error
+    libp2pService.processInChunksService.process = jest.fn()
     class RemotePeerEventDetail {
       peerId: string
 
@@ -137,6 +139,10 @@ describe('Connections manager', () => {
 
     // Peer connected
     await connectionsManagerService.init()
+    await connectionsManagerService.launchCommunity({
+      community,
+      network: { peerId: userIdentity.peerId, hiddenService: userIdentity.hiddenService },
+    })
     libp2pService.connectedPeers.set(peerId.toString(), DateTime.utc().valueOf())
 
     // Peer disconnected
@@ -145,11 +151,16 @@ describe('Connections manager', () => {
       remotePeer: new RemotePeerEventDetail(peerId.toString()),
       remoteAddr: new RemotePeerEventDetail(remoteAddr),
     }
+    await waitForExpect(async () => {
+      expect(libp2pService.libp2pInstance).not.toBeUndefined()
+    }, 2_000)
     libp2pService.libp2pInstance?.dispatchEvent(
       new CustomEvent('peer:disconnect', { detail: peerDisconectEventDetail })
     )
+    await waitForExpect(async () => {
+      expect(libp2pService.connectedPeers.size).toEqual(0)
+    }, 2000)
 
-    expect(libp2pService.connectedPeers.size).toEqual(0)
     await waitForExpect(async () => {
       expect(await localDbService.get(LocalDBKeys.PEERS)).not.toBeNull()
     }, 2000)
@@ -166,7 +177,6 @@ describe('Connections manager', () => {
     const spyOnDestroyHiddenService = jest.spyOn(tor, 'destroyHiddenService')
     await connectionsManagerService.init()
     const network = await connectionsManagerService.getNetwork()
-    console.log('network', network)
     expect(network.hiddenService.onionAddress.split('.')[0]).toHaveLength(56)
     expect(network.hiddenService.privateKey).toHaveLength(99)
     const peerId = await PeerId.createFromJSON(network.peerId)

packages/backend/src/nest/connections-manager/connections-manager.service.ts

@@ -1,69 +1,69 @@
+import { peerIdFromKeys } from '@libp2p/peer-id'
 import { Inject, Injectable, OnModuleInit } from '@nestjs/common'
 import { Crypto } from '@peculiar/webcrypto'
-import { Agent } from 'https'
-import fs from 'fs'
-import path from 'path'
-import { peerIdFromKeys } from '@libp2p/peer-id'
-import { setEngine, CryptoEngine } from 'pkijs'
 import { EventEmitter } from 'events'
+import fs from 'fs'
 import getPort from 'get-port'
+import { Agent } from 'https'
+import path from 'path'
 import PeerId from 'peer-id'
+import { CryptoEngine, setEngine } from 'pkijs'
 import { getLibp2pAddressesFromCsrs, removeFilesFromDir } from '../common/utils'
 
+import { LazyModuleLoader } from '@nestjs/core'
+import { createLibp2pAddress, isPSKcodeValid, p2pAddressesToPairs } from '@quiet/common'
+import { CertFieldsTypes, getCertFieldValue, loadCertificate } from '@quiet/identity'
 import {
-  GetMessagesPayload,
   ChannelMessageIdsResponse,
-  type DeleteChannelResponse,
+  ChannelSubscribedPayload,
   ChannelsReplicatedPayload,
   Community,
-  CommunityId,
+  CommunityMetadata,
+  CommunityOwnership,
   ConnectionProcessInfo,
   CreateChannelPayload,
   CreateChannelResponse,
+  CreateNetworkPayload,
   DeleteFilesFromChannelSocketPayload,
   DownloadStatus,
   ErrorMessages,
   FileMetadata,
-  MessagesLoadedPayload,
+  GetMessagesPayload,
   InitCommunityPayload,
+  MessagesLoadedPayload,
   NetworkDataPayload,
   NetworkInfo,
   NetworkStats,
-  type SavedOwnerCertificatePayload,
   PushNotificationPayload,
-  RegisterOwnerCertificatePayload,
   RemoveDownloadStatus,
+  SaveCSRPayload,
   SendCertificatesResponse,
   SendMessagePayload,
-  ChannelSubscribedPayload,
+  ServerInvitationData,
   SocketActionTypes,
-  StorePeerListPayload,
   UploadFilePayload,
-  PeerId as PeerIdType,
-  SaveCSRPayload,
-  CommunityMetadata,
-  type PermsData,
+  type DeleteChannelResponse,
+  type SavedOwnerCertificatePayload,
   type UserProfile,
   type UserProfilesStoredEvent,
 } from '@quiet/types'
+import Logger from '../common/logger'
 import { CONFIG_OPTIONS, QUIET_DIR, SERVER_IO_PROVIDER, SOCKS_PROXY_AGENT } from '../const'
-import { ConfigOptions, GetPorts, ServerIoProviderTypes } from '../types'
-import { SocketService } from '../socket/socket.service'
-import { RegistrationService } from '../registration/registration.service'
-import { LocalDbService } from '../local-db/local-db.service'
-import { StorageService } from '../storage/storage.service'
-import { ServiceState, TorInitState } from './connections-manager.types'
 import { Libp2pService } from '../libp2p/libp2p.service'
-import { Tor } from '../tor/tor.service'
-import { LocalDBKeys } from '../local-db/local-db.types'
 import { Libp2pEvents, Libp2pNodeParams } from '../libp2p/libp2p.types'
+import { LocalDbService } from '../local-db/local-db.service'
+import { LocalDBKeys } from '../local-db/local-db.types'
+import { RegistrationService } from '../registration/registration.service'
 import { RegistrationEvents } from '../registration/registration.types'
-import { StorageEvents } from '../storage/storage.types'
-import { LazyModuleLoader } from '@nestjs/core'
-import Logger from '../common/logger'
 import { emitError } from '../socket/socket.errors'
-import { createLibp2pAddress, isPSKcodeValid } from '@quiet/common'
-import { CertFieldsTypes, createRootCA, getCertFieldValue, loadCertificate } from '@quiet/identity'
+import { SocketService } from '../socket/socket.service'
+import { StorageService } from '../storage/storage.service'
+import { StorageEvents } from '../storage/storage.types'
+import { ServerProxyService } from '../storageServerProxy/storageServerProxy.service'
+import { ServerStoredCommunityMetadata } from '../storageServerProxy/storageServerProxy.types'
+import { Tor } from '../tor/tor.service'
+import { ConfigOptions, GetPorts, ServerIoProviderTypes } from '../types'
+import { ServiceState, TorInitState } from './connections-manager.types'
 
 @Injectable()
 export class ConnectionsManagerService extends EventEmitter implements OnModuleInit {
@@ -81,6 +81,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
     @Inject(SOCKS_PROXY_AGENT) public readonly socksProxyAgent: Agent,
     private readonly socketService: SocketService,
     private readonly registrationService: RegistrationService,
+    private readonly storageServerProxyService: ServerProxyService,
     private readonly localDbService: LocalDbService,
     private readonly storageService: StorageService,
     private readonly tor: Tor,
@@ -392,15 +393,51 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
     return community
   }
 
+  public async downloadCommunityData(inviteData: ServerInvitationData) {
+    this.logger('Downloading invite data', inviteData)
+    this.storageServerProxyService.setServerAddress(inviteData.serverAddress)
+    let downloadedData: ServerStoredCommunityMetadata
+    try {
+      downloadedData = await this.storageServerProxyService.downloadData(inviteData.cid)
+    } catch (e) {
+      this.logger.error(`Downloading community data failed`, e)
+      return
+    }
+    return {
+      psk: downloadedData.psk,
+      peers: downloadedData.peerList,
+      ownerOrbitDbIdentity: downloadedData.ownerOrbitDbIdentity,
+    }
+  }
+
   public async joinCommunity(payload: InitCommunityPayload): Promise<Community | undefined> {
     this.logger('Joining community: peers:', payload.peers)
+    let metadata = {
+      psk: payload.psk,
+      peers: payload.peers,
+      ownerOrbitDbIdentity: payload.ownerOrbitDbIdentity,
+    }
 
-    if (!payload.peers || payload.peers.length === 0) {
+    const inviteData = payload.inviteData
+    if (inviteData) {
+      const downloadedData = await this.downloadCommunityData(inviteData)
+      if (!downloadedData) {
+        emitError(this.serverIoProvider.io, {
+          type: SocketActionTypes.LAUNCH_COMMUNITY,
+          message: ErrorMessages.STORAGE_SERVER_CONNECTION_FAILED,
+        })
+        return
+      }
+      metadata = downloadedData
+    }
+    this.logger('Joining community: metadata:', metadata)
+
+    if (!metadata.peers || metadata.peers.length === 0) {
       this.logger.error('Joining community: Peers required')
       return
     }
 
-    if (!payload.psk || !isPSKcodeValid(payload.psk)) {
+    if (!metadata.psk || !isPSKcodeValid(metadata.psk)) {
       this.logger.error('Joining community: Libp2p PSK is not valid')
       emitError(this.serverIoProvider.io, {
         type: SocketActionTypes.LAUNCH_COMMUNITY,
@@ -410,7 +447,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
       return
     }
 
-    if (!payload.ownerOrbitDbIdentity) {
+    if (!metadata.ownerOrbitDbIdentity) {
       this.logger.error('Joining community: ownerOrbitDbIdentity is not valid')
       emitError(this.serverIoProvider.io, {
         type: SocketActionTypes.LAUNCH_COMMUNITY,
@@ -424,9 +461,9 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI
 
     const community = {
       id: payload.id,
-      peerList: [...new Set([localAddress, ...payload.peers])],
-      psk: payload.psk,
-      ownerOrbitDbIdentity: payload.ownerOrbitDbIdentity,
+      peerList: [...new Set([localAddress, ...metadata.peers])],
+      psk: metadata.psk,
+      ownerOrbitDbIdentity: metadata.ownerOrbitDbIdentity,
     }
 
     const network = {

packages/backend/src/nest/libp2p/process-in-chunks.service.ts

@@ -13,7 +13,7 @@ export class ProcessInChunksService<T> extends EventEmitter {
     super()
   }
 
-  public init(data: T[], processItem: (arg: T) => Promise<any>, chunkSize: number = DEFAULT_CHUNK_SIZE) {
+  public init(data: T[] = [], processItem: (arg: T) => Promise<any>, chunkSize: number = DEFAULT_CHUNK_SIZE) {
     this.data = data
     this.processItem = processItem
     this.chunkSize = chunkSize

packages/backend/src/nest/registration/registration.functions.ts

@@ -64,7 +64,6 @@ export const extractPendingCsrs = async (payload: { csrs: string[]; certificates
       pendingCsrs.push(csr)
     }
   }
-  logger('DuplicatedCertBug', { parsedUniqueCsrs, pendingNames, certNames })
   return pendingCsrs
 }