diff --git a/packages/backend/package-lock.json b/packages/backend/package-lock.json index 3a695a2a99..43868588fa 100644 --- a/packages/backend/package-lock.json +++ b/packages/backend/package-lock.json @@ -27,6 +27,7 @@ "events": "^3.2.0", "express": "^4.17.1", "fetch-retry": "^6.0.0", + "fastq": "^1.17.1", "get-port": "^5.1.1", "go-ipfs": "npm:mocked-go-ipfs@0.17.0", "http-server": "^0.12.3", @@ -10817,6 +10818,23 @@ "node": ">= 4.9.1" } }, + "node_modules/fastq": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fastq/node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, "node_modules/fb-watchman": { "version": "2.0.1", "license": "Apache-2.0", @@ -30789,6 +30807,21 @@ "version": "1.0.16", "dev": true }, + "fastq": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", + "requires": { + "reusify": "^1.0.4" + }, + "dependencies": { + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==" + } + } + }, "fb-watchman": { "version": "2.0.1", "requires": { diff --git a/packages/backend/package.json b/packages/backend/package.json index ce170d5e09..8a443cf572 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -109,6 +109,7 @@ "events": "^3.2.0", "express": "^4.17.1", "fetch-retry": "^6.0.0", + "fastq": "^1.17.1", "get-port": "^5.1.1", "go-ipfs": "npm:mocked-go-ipfs@0.17.0", "http-server": "^0.12.3", diff --git a/packages/backend/src/nest/connections-manager/connections-manager.service.tor.spec.ts b/packages/backend/src/nest/connections-manager/connections-manager.service.tor.spec.ts index 08f6614bad..9f448837d9 100644 --- a/packages/backend/src/nest/connections-manager/connections-manager.service.tor.spec.ts +++ b/packages/backend/src/nest/connections-manager/connections-manager.service.tor.spec.ts @@ -30,6 +30,7 @@ import waitForExpect from 'wait-for-expect' import { Libp2pEvents } from '../libp2p/libp2p.types' import { sleep } from '../common/sleep' import { createLibp2pAddress } from '@quiet/common' +import { lib } from 'crypto-js' jest.setTimeout(100_000) @@ -215,8 +216,8 @@ describe('Connections manager', () => { await sleep(5000) // It looks LibP2P dials peers initially when it's started and // then IPFS service dials peers again when started, thus - // peersCount * 2 - expect(spyOnDial).toHaveBeenCalledTimes(peersCount * 2) + // peersCount-1 * 2 because we don't dial ourself (the first peer in the list) + expect(spyOnDial).toHaveBeenCalledTimes((peersCount - 1) * 2) // Temporary fix for hanging test - websocketOverTor doesn't have abortController await sleep(5000) }) diff --git a/packages/backend/src/nest/connections-manager/connections-manager.service.ts b/packages/backend/src/nest/connections-manager/connections-manager.service.ts index d96a878b46..fef770f7fb 100644 --- a/packages/backend/src/nest/connections-manager/connections-manager.service.ts +++ b/packages/backend/src/nest/connections-manager/connections-manager.service.ts @@ -517,7 +517,7 @@ export class ConnectionsManagerService extends EventEmitter implements OnModuleI agent: this.socksProxyAgent, localAddress: this.libp2pService.createLibp2pAddress(onionAddress, peerId.toString()), targetPort: this.ports.libp2pHiddenService, - peers: peers ?? [], + peers: peers ? peers.slice(1) : [], psk: Libp2pService.generateLibp2pPSK(community.psk).fullKey, } await this.libp2pService.createInstance(params) diff --git a/packages/backend/src/nest/libp2p/libp2p.service.spec.ts b/packages/backend/src/nest/libp2p/libp2p.service.spec.ts index 13ebb5f1ed..91d9d177c4 100644 --- a/packages/backend/src/nest/libp2p/libp2p.service.spec.ts +++ b/packages/backend/src/nest/libp2p/libp2p.service.spec.ts @@ -8,7 +8,7 @@ import { Libp2pEvents, Libp2pNodeParams } from './libp2p.types' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import validator from 'validator' import waitForExpect from 'wait-for-expect' -import { ProcessInChunksService } from './process-in-chunks.service' +import { DEFAULT_NUM_TRIES, ProcessInChunksService } from './process-in-chunks.service' describe('Libp2pService', () => { let module: TestingModule @@ -93,10 +93,12 @@ describe('Libp2pService', () => { await libp2pService.createInstance(params) expect(libp2pService.libp2pInstance).not.toBeNull() // @ts-expect-error processItem is private - const dialPeerSpy = jest.spyOn(processInChunks, 'processItem') + const processItemSpy = jest.spyOn(processInChunks, 'processItem') + const dialSpy = jest.spyOn(libp2pService.libp2pInstance!, 'dial') libp2pService.emit(Libp2pEvents.DIAL_PEERS, addresses) await waitForExpect(async () => { - expect(dialPeerSpy).toBeCalledTimes(1) + expect(processItemSpy).toBeCalledTimes(2 * DEFAULT_NUM_TRIES) + expect(dialSpy).toBeCalledTimes(1) }) }) }) diff --git a/packages/backend/src/nest/libp2p/process-in-chunks.service.ts b/packages/backend/src/nest/libp2p/process-in-chunks.service.ts index b87892ea3d..e57e681227 100644 --- a/packages/backend/src/nest/libp2p/process-in-chunks.service.ts +++ b/packages/backend/src/nest/libp2p/process-in-chunks.service.ts @@ -1,56 +1,85 @@ import { EventEmitter } from 'events' +import fastq from 'fastq' +import type { queue, done } from 'fastq' + import Logger from '../common/logger' const DEFAULT_CHUNK_SIZE = 10 +export const DEFAULT_NUM_TRIES = 2 + +type ProcessTask = { + data: T + tries: number +} export class ProcessInChunksService extends EventEmitter { private isActive: boolean - private data: T[] + private data: Set = new Set() private chunkSize: number + private taskQueue: queue> private processItem: (arg: T) => Promise private readonly logger = Logger(ProcessInChunksService.name) constructor() { super() } - public init(data: T[] = [], processItem: (arg: T) => Promise, chunkSize: number = DEFAULT_CHUNK_SIZE) { - this.data = data + public init(data: T[], processItem: (arg: T) => Promise, chunkSize: number = DEFAULT_CHUNK_SIZE) { + this.logger(`Initializing process-in-chunks.service with peers ${JSON.stringify(data, null, 2)}`) this.processItem = processItem this.chunkSize = chunkSize + this.taskQueue = fastq(this, this.processOneItem, this.chunkSize) + this.updateData(data) + this.addToTaskQueue() } - updateData(items: T[]) { + public updateData(items: T[]) { this.logger(`Updating data with ${items.length} items`) - this.data = [...new Set(this.data.concat(items))] + this.taskQueue.pause() + items.forEach(item => this.data.add(item)) + this.addToTaskQueue() } - public async processOneItem() { - const toProcess = this.data.shift() - if (toProcess) { - try { - await this.processItem(toProcess) - } catch (e) { - this.logger(`Processing ${toProcess} failed, message:`, e.message) - } finally { - process.nextTick(async () => { - await this.processOneItem() - }) + private addToTaskQueue() { + this.logger(`Adding ${this.data.size} items to the task queue`) + for (const item of this.data) { + if (item) { + this.logger(`Adding data ${item} to the task queue`) + this.data.delete(item) + try { + this.taskQueue.push({ data: item, tries: 0 } as ProcessTask) + } catch (e) { + this.logger.error(`Error occurred while adding new task for item ${item} to the queue`, e) + this.data.add(item) + } } } } - public async process() { - this.logger(`Processing ${this.data.length} items`) - for (let i = 0; i < this.chunkSize; i++) { - // Do not wait for this promise as items should be processed simultineously - void this.processOneItem() + public async processOneItem(task: ProcessTask) { + try { + this.logger(`Processing task with data ${task.data}`) + await this.processItem(task.data) + } catch (e) { + this.logger.error(`Processing task with data ${task.data} failed`, e) + if (task.tries + 1 < DEFAULT_NUM_TRIES) { + this.logger(`Will try to re-attempt task with data ${task.data}`) + this.taskQueue.push({ ...task, tries: task.tries + 1 }) + } + } finally { + this.logger(`Done attempting to process task with data ${task.data}`) } } + public async process() { + this.logger(`Processing ${this.taskQueue.length} items`) + this.taskQueue.resume() + } + public stop() { if (this.isActive) { this.logger('Stopping initial dial') this.isActive = false + this.taskQueue.pause() } } } diff --git a/packages/backend/src/nest/libp2p/process-in-chunks.spec.ts b/packages/backend/src/nest/libp2p/process-in-chunks.spec.ts index 38979c1cf9..ce751afdf3 100644 --- a/packages/backend/src/nest/libp2p/process-in-chunks.spec.ts +++ b/packages/backend/src/nest/libp2p/process-in-chunks.spec.ts @@ -27,7 +27,7 @@ describe('ProcessInChunks', () => { processInChunks.init(['a', 'b', 'c', 'd'], mockProcessItem) await processInChunks.process() await waitForExpect(() => { - expect(mockProcessItem).toBeCalledTimes(4) + expect(mockProcessItem).toBeCalledTimes(6) }) }) @@ -43,7 +43,7 @@ describe('ProcessInChunks', () => { processInChunks.updateData(['e', 'f']) await processInChunks.process() await waitForExpect(() => { - expect(mockProcessItem).toBeCalledTimes(4) + expect(mockProcessItem).toBeCalledTimes(5) }) }) @@ -60,7 +60,7 @@ describe('ProcessInChunks', () => { processInChunks.init(['a', 'b', 'c', 'd'], mockProcessItem, chunkSize) await processInChunks.process() await waitForExpect(() => { - expect(mockProcessItem).toBeCalledTimes(4) + expect(mockProcessItem).toBeCalledTimes(2) }) }) diff --git a/packages/backend/src/nest/local-db/local-db.service.ts b/packages/backend/src/nest/local-db/local-db.service.ts index 6c85552ac8..fd36ed80d8 100644 --- a/packages/backend/src/nest/local-db/local-db.service.ts +++ b/packages/backend/src/nest/local-db/local-db.service.ts @@ -95,7 +95,21 @@ export class LocalDbService { } } - public async getSortedPeers(peers: string[] = []): Promise { + public async getSortedPeers( + peers?: string[] | undefined, + includeLocalPeerAddress: boolean = true + ): Promise { + if (!peers) { + const currentCommunity = await this.getCurrentCommunity() + if (!currentCommunity) { + throw new Error('No peers were provided and no community was found to extract peers from') + } + peers = currentCommunity.peerList + if (!peers) { + throw new Error('No peers provided and no peers found on current stored community') + } + } + const peersStats = (await this.get(LocalDBKeys.PEERS)) || {} const stats: NetworkStats[] = Object.values(peersStats) const network = await this.getNetworkInfo() @@ -103,9 +117,9 @@ export class LocalDbService { if (network) { const localPeerAddress = createLibp2pAddress(network.hiddenService.onionAddress, network.peerId.id) this.logger('Local peer', localPeerAddress) - return filterAndSortPeers(peers, stats, localPeerAddress) + return filterAndSortPeers(peers, stats, localPeerAddress, includeLocalPeerAddress) } else { - return filterAndSortPeers(peers, stats) + return filterAndSortPeers(peers, stats, undefined, includeLocalPeerAddress) } } diff --git a/packages/backend/src/nest/socket/socket.service.ts b/packages/backend/src/nest/socket/socket.service.ts index 1f61e201c3..48b51151d5 100644 --- a/packages/backend/src/nest/socket/socket.service.ts +++ b/packages/backend/src/nest/socket/socket.service.ts @@ -22,6 +22,7 @@ import { type MessagesLoadedPayload, type NetworkInfo, CreateNetworkPayload, + CommunityOwnership, } from '@quiet/types' import EventEmitter from 'events' import { CONFIG_OPTIONS, SERVER_IO_PROVIDER } from '../const' diff --git a/packages/common/src/sortPeers.ts b/packages/common/src/sortPeers.ts index 682111c1f2..634a79cfaf 100644 --- a/packages/common/src/sortPeers.ts +++ b/packages/common/src/sortPeers.ts @@ -1,5 +1,4 @@ import { type NetworkStats } from '@quiet/types' -import { isDefined } from './helpers' import { filterValidAddresses } from './libp2p' /** @@ -14,9 +13,11 @@ This is the very simple algorithm for evaluating the most wanted peers. export const filterAndSortPeers = ( peersAddresses: string[], stats: NetworkStats[], - localPeerAddress?: string + localPeerAddress?: string, + includeLocalPeerAddress: boolean = true ): string[] => { peersAddresses = filterValidAddresses(peersAddresses) + const currentlyConnected = [...stats].filter(peer => peer.connectionTime === 0) const lastSeenSorted = [...stats].sort((a, b) => { return b.lastSeen - a.lastSeen }) @@ -24,7 +25,7 @@ export const filterAndSortPeers = ( return b.connectionTime - a.connectionTime }) - const mostWantedPeers: NetworkStats[] = [] + const mostWantedPeers: NetworkStats[] = currentlyConnected for (let i = 0; i < stats.length; i++) { const peerOne = lastSeenSorted[i] @@ -39,22 +40,28 @@ export const filterAndSortPeers = ( } } - const peerList = mostWantedPeers.map(peerId => { - return peersAddresses.find(peerAddress => { + const peerSet: Set = new Set() + if (includeLocalPeerAddress && localPeerAddress) { + peerSet.add(localPeerAddress) + } + + mostWantedPeers.forEach(peer => { + const found = peersAddresses.find(peerAddress => { const id = peerAddress.split('/')[7] - if (id === peerId.peerId) { + if (id === peer.peerId) { peersAddresses.splice(peersAddresses.indexOf(peerAddress), 1) return true } }) + if (found && found !== '') { + peerSet.add(found) + } + }) + peersAddresses.forEach(peerAddress => { + if (!peerSet.has(peerAddress)) { + peerSet.add(peerAddress) + } }) - return [ - ...new Set([ - localPeerAddress, // Set local peer as first - ...peerList.concat(peersAddresses), - ]), - ] - .filter(address => address !== null && address !== '') - .filter(isDefined) + return [...peerSet] } diff --git a/packages/common/src/tests.ts b/packages/common/src/tests.ts index 603c9342df..6c4791f4bb 100644 --- a/packages/common/src/tests.ts +++ b/packages/common/src/tests.ts @@ -52,3 +52,12 @@ export function getValidInvitationUrlTestData { +// return { +// shareUrl: () => composeInvitationShareUrl(data), +// deepUrl: () => composeInvitationDeepUrl(data), +// code: () => composeInvitationShareUrl(data).split(QUIET_JOIN_PAGE + '#')[1], +// data: data, +// } +// } diff --git a/packages/desktop/CHANGELOG.md b/packages/desktop/CHANGELOG.md index 59c0741d0f..e5ad80e5c2 100644 --- a/packages/desktop/CHANGELOG.md +++ b/packages/desktop/CHANGELOG.md @@ -1,20 +1,28 @@ [unreleased] -* Refactored package.json to have consistent license "GPL-3.0-or-later" - -# Refactorings: +# New features: -* Use ack for CREATE_NETWORK and simplify +# Refactorings: -# Fixes +# Fixes: -* Allow JPEG and GIF files as profile photos ([#2332](https://github.com/TryQuiet/quiet/issues/2332)) -* Fixes issues with recreating general channel when deleted while offline ([#2334](https://github.com/TryQuiet/quiet/issues/2334)) +[2.2.0] -# New features +# New features: * Add utilities for emoji detection in messages and make all-emoji message larger font size ([#519](https://github.com/TryQuiet/quiet/issues/519)) +# Refactorings: + +* Use ack for CREATE_NETWORK and simplify +* Move Community model to the backend + +# Fixes: + +* Allow JPEG and GIF files as profile photos ([#2332](https://github.com/TryQuiet/quiet/issues/2332)) +* Fix issues with recreating general channel when deleted while offline ([#2334](https://github.com/TryQuiet/quiet/issues/2334)) +* Fix package.json license inconsistency + [2.1.2] # Refactorings: @@ -274,4 +282,3 @@ * C4 for Quiet architecture. Context and Container diagrams. * Invite tab as default in settings - diff --git a/packages/desktop/package-lock.json b/packages/desktop/package-lock.json index 4294790b37..c9e049edb0 100644 --- a/packages/desktop/package-lock.json +++ b/packages/desktop/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quiet/desktop", - "version": "2.2.0-alpha.0", + "version": "2.2.0-alpha.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@quiet/desktop", - "version": "2.2.0-alpha.0", + "version": "2.2.0-alpha.1", "license": "GPL-3.0-or-later", "dependencies": { "@electron/remote": "^2.0.8", diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 0784f8c19a..f8bb9b22ca 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -80,7 +80,7 @@ }, "homepage": "https://github.com/TryQuiet", "@comment version": "To build new version for specific platform, just replace platform in version tag to one of following linux, mac, windows", - "version": "2.2.0-alpha.0", + "version": "2.2.0-alpha.1", "description": "Decentralized team chat", "main": "dist/main/main.js", "scripts": { diff --git a/packages/mobile/CHANGELOG.md b/packages/mobile/CHANGELOG.md index 59c0741d0f..e5ad80e5c2 100644 --- a/packages/mobile/CHANGELOG.md +++ b/packages/mobile/CHANGELOG.md @@ -1,20 +1,28 @@ [unreleased] -* Refactored package.json to have consistent license "GPL-3.0-or-later" - -# Refactorings: +# New features: -* Use ack for CREATE_NETWORK and simplify +# Refactorings: -# Fixes +# Fixes: -* Allow JPEG and GIF files as profile photos ([#2332](https://github.com/TryQuiet/quiet/issues/2332)) -* Fixes issues with recreating general channel when deleted while offline ([#2334](https://github.com/TryQuiet/quiet/issues/2334)) +[2.2.0] -# New features +# New features: * Add utilities for emoji detection in messages and make all-emoji message larger font size ([#519](https://github.com/TryQuiet/quiet/issues/519)) +# Refactorings: + +* Use ack for CREATE_NETWORK and simplify +* Move Community model to the backend + +# Fixes: + +* Allow JPEG and GIF files as profile photos ([#2332](https://github.com/TryQuiet/quiet/issues/2332)) +* Fix issues with recreating general channel when deleted while offline ([#2334](https://github.com/TryQuiet/quiet/issues/2334)) +* Fix package.json license inconsistency + [2.1.2] # Refactorings: @@ -274,4 +282,3 @@ * C4 for Quiet architecture. Context and Container diagrams. * Invite tab as default in settings - diff --git a/packages/mobile/android/app/build.gradle b/packages/mobile/android/app/build.gradle index 021ad4583c..d74a8bbe60 100644 --- a/packages/mobile/android/app/build.gradle +++ b/packages/mobile/android/app/build.gradle @@ -192,8 +192,8 @@ android { applicationId "com.quietmobile" minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion - versionCode 412 - versionName "2.2.0-alpha.0" + versionCode 413 + versionName "2.2.0-alpha.1" resValue "string", "build_config_package", "com.quietmobile" testBuildType System.getProperty('testBuildType', 'debug') testInstrumentationRunner 'androidx.test.runner.AndroidJUnitRunner' diff --git a/packages/mobile/ios/Quiet/Info.plist b/packages/mobile/ios/Quiet/Info.plist index 1140a0beac..01a2cc8c61 100644 --- a/packages/mobile/ios/Quiet/Info.plist +++ b/packages/mobile/ios/Quiet/Info.plist @@ -34,7 +34,7 @@ CFBundleVersion - 369 + 370 ITSAppUsesNonExemptEncryption LSRequiresIPhoneOS diff --git a/packages/mobile/ios/QuietTests/Info.plist b/packages/mobile/ios/QuietTests/Info.plist index 7f5a67eb0a..167b0126a2 100644 --- a/packages/mobile/ios/QuietTests/Info.plist +++ b/packages/mobile/ios/QuietTests/Info.plist @@ -19,6 +19,6 @@ CFBundleSignature ???? CFBundleVersion - 369 + 370 diff --git a/packages/mobile/package-lock.json b/packages/mobile/package-lock.json index 210e059f3d..5f0c2a9a1a 100644 --- a/packages/mobile/package-lock.json +++ b/packages/mobile/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quiet/mobile", - "version": "2.2.0-alpha.0", + "version": "2.2.0-alpha.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@quiet/mobile", - "version": "2.2.0-alpha.0", + "version": "2.2.0-alpha.1", "dependencies": { "@peculiar/webcrypto": "^1.4.3", "@react-native-clipboard/clipboard": "^1.13.2", diff --git a/packages/mobile/package.json b/packages/mobile/package.json index b29f0f248d..3f052532e6 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@quiet/mobile", - "version": "2.2.0-alpha.0", + "version": "2.2.0-alpha.1", "scripts": { "build": "tsc -p tsconfig.build.json --noEmit", "storybook-android": "ENVFILE=.env.storybook react-native run-android --mode=storybookDebug --appIdSuffix=storybook.debug",