Skip to content

Commit

Permalink
Batch download images during initial load (#2374)
Browse files Browse the repository at this point in the history 
* add batchDownloadFiles

* process download in batches
  • Loading branch information
@ratik21
ratik21 authored Oct 21, 2024
1 parent 0bf1466 commit e1a40f1
Show file tree
Hide file tree
Showing 4 changed files with 151 additions and 96 deletions.
8 changes: 8 additions & 0 deletions src/lib/chat/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ export interface IChatClient {
getRoomIdForAlias: (alias: string) => Promise<string | undefined>;
uploadFile(file: File): Promise<string>;
downloadFile(fileUrl: string): Promise<any>;
batchDownloadFiles(fileUrls: string[], batchSize: number): Promise<{ [fileUrl: string]: string }>;
editProfile(profileInfo: MatrixProfileInfo): Promise<void>;
getAccessToken(): string | null;
mxcUrlToHttp(mxcUrl: string): string;
Expand Down Expand Up @@ -389,6 +390,13 @@ export async function downloadFile(fileUrl: string) {
return chat.get().matrix.downloadFile(fileUrl);
}

export async function batchDownloadFiles(
fileUrls: string[],
batchSize: number = 15
): Promise<{ [fileUrl: string]: string }> {
return chat.get().matrix.batchDownloadFiles(fileUrls, batchSize);
}

export async function editProfile(profileInfo: MatrixProfileInfo) {
return chat.get().matrix.editProfile(profileInfo);
}
Expand Down
36 changes: 36 additions & 0 deletions src/lib/chat/matrix-client.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,42 @@ export class MatrixClient implements IChatClient {
return URL.createObjectURL(blob);
}

async batchDownloadFiles(fileUrls: string[], batchSize: number = 15): Promise<{ [fileUrl: string]: string }> {
// Helper function to download a single file
const downloadFileWithFallback = async (fileUrl: string) => {
try {
const downloadedUrl = await this.downloadFile(fileUrl);
return { [fileUrl]: downloadedUrl };
} catch (error) {
console.log(`Error downloading file ${fileUrl}:`, error);
return { [fileUrl]: '' }; // If the download fails, return an empty string as a fallback
}
};

const downloadResultsMap = {};

// Split the file URLs into batches
for (let i = 0; i < fileUrls.length; i += batchSize) {
const batch = fileUrls.slice(i, i + batchSize);

console.log(`Downloading batch ${i / batchSize + 1} of ${Math.ceil(fileUrls.length / batchSize)} `, batch);

// Download the current batch of files concurrently
const batchResultsArray: Array<{ [fileUrl: string]: string }> = await Promise.all(
batch.map((fileUrl) => downloadFileWithFallback(fileUrl))
);

console.log(`Download for batch ${i / batchSize + 1} complete: `, batchResultsArray);

// Merge the results of the current batch into the overall map
batchResultsArray.forEach((result) => {
Object.assign(downloadResultsMap, result);
});
}

return downloadResultsMap;
}

async editProfile(profileInfo: MatrixProfileInfo = {}) {
await this.waitForConnection();
if (profileInfo.displayName) {
Expand Down
176 changes: 92 additions & 84 deletions src/store/channels-list/saga.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { testSaga } from 'redux-saga-test-plan';
import { call } from 'redux-saga/effects';
import * as matchers from 'redux-saga-test-plan/matchers';
import { chat, downloadFile, getRoomTags } from '../../lib/chat';
import { batchDownloadFiles, chat, getRoomTags } from '../../lib/chat';

import {
fetchConversations,
Expand Down Expand Up @@ -30,7 +30,6 @@ import { getZEROUsers } from './api';
import { mapAdminUserIdToZeroUserId, mapChannelMembers } from './utils';
import { openFirstConversation } from '../channels/saga';
import { getUserReadReceiptPreference } from '../user-profile/saga';
import cloneDeep from 'lodash/cloneDeep';

const mockConversation = (id: string) => ({
id: `conversation_${id}`,
Expand Down Expand Up @@ -312,60 +311,82 @@ describe('channels list saga', () => {
});

describe(mapToZeroUsers, () => {
const rooms = [
{
id: 'room-1',
otherMembers: [],
memberHistory: [
{ matrixId: 'matrix-id-1', userId: 'matrix-id-1' },
{ matrixId: 'matrix-id-2', userId: 'matrix-id-2' },
],
messages: [],
moderatorIds: [],
},
{
id: 'room-2',
otherMembers: [],
memberHistory: [
{ matrixId: 'matrix-id-3', userId: 'matrix-id-3' },
],
messages: [],
moderatorIds: [],
},
] as any;
let rooms, zeroUsers;

const zeroUsers = [
{
userId: 'user-1',
matrixId: 'matrix-id-1',
profileId: 'profile-1',
firstName: 'first-1',
lastName: 'last-1',
primaryZID: 'primary-zid-1',
displaySubHandle: 'primary-zid-1',
profileImage: 'profile-image-url-1',
},
{
userId: 'user-2',
matrixId: 'matrix-id-2',
profileId: 'profile-2',
firstName: 'first-2',
lastName: 'last-2',
primaryZID: 'primary-zid-2',
displaySubHandle: 'primary-zid-2',
profileImage: 'profile-image-url-2',
},
{
userId: 'user-3',
matrixId: 'matrix-id-3',
profileId: 'profile-3',
firstName: 'first-3',
lastName: 'last-3',
primaryZID: '',
displaySubHandle: '',
profileImage: 'profile-image-url-3',
},
] as any;
beforeEach(() => {
rooms = [
{
id: 'room-1',
otherMembers: [],
memberHistory: [
{ matrixId: 'matrix-id-1', userId: 'matrix-id-1' },
{ matrixId: 'matrix-id-2', userId: 'matrix-id-2' },
],
messages: [],
moderatorIds: [],
},
{
id: 'room-2',
otherMembers: [],
memberHistory: [
{ matrixId: 'matrix-id-3', userId: 'matrix-id-3' },
],
messages: [],
moderatorIds: [],
},
] as any;

zeroUsers = [
{
userId: 'user-1',
matrixId: 'matrix-id-1',
profileId: 'profile-1',
firstName: 'first-1',
lastName: 'last-1',
primaryZID: 'primary-zid-1',
displaySubHandle: 'primary-zid-1',
profileImage: 'mxc://profile-image-url-1',
},
{
userId: 'user-2',
matrixId: 'matrix-id-2',
profileId: 'profile-2',
firstName: 'first-2',
lastName: 'last-2',
primaryZID: 'primary-zid-2',
displaySubHandle: 'primary-zid-2',
profileImage: 'mxc://profile-image-url-2',
},
{
userId: 'user-3',
matrixId: 'matrix-id-3',
profileId: 'profile-3',
firstName: 'first-3',
lastName: 'last-3',
primaryZID: '',
displaySubHandle: '',
profileImage: 'mxc://profile-image-url-3',
},
] as any;
});

const getProviders = (zeroUsers) => {
return [
[call(getZEROUsers, ['matrix-id-1', 'matrix-id-2', 'matrix-id-3']), zeroUsers],
[
call(batchDownloadFiles, [
'mxc://profile-image-url-1',
'mxc://profile-image-url-2',
'mxc://profile-image-url-3',
]),
{
'mxc://profile-image-url-1': 'blob://profile-image-url-1',
'mxc://profile-image-url-2': 'blob://profile-image-url-2',
'mxc://profile-image-url-3': 'blob://profile-image-url-3',
},
],
] as any;
};

it('calls getZEROUsers by merging all matrixIds', async () => {
await expectSaga(mapToZeroUsers, rooms)
Expand All @@ -374,7 +395,7 @@ describe('channels list saga', () => {
.run();
});

it('creates map for zero users and downloads profile images after fetching from api', async () => {
it('creates map for zero users and downloads profile images', async () => {
const expectedMap = {
'matrix-id-1': {
userId: 'user-1',
Expand All @@ -384,7 +405,7 @@ describe('channels list saga', () => {
lastName: 'last-1',
primaryZID: 'primary-zid-1',
displaySubHandle: 'primary-zid-1',
profileImage: 'profile-image-url-1',
profileImage: 'blob://profile-image-url-1',
},
'matrix-id-2': {
userId: 'user-2',
Expand All @@ -394,7 +415,7 @@ describe('channels list saga', () => {
lastName: 'last-2',
primaryZID: 'primary-zid-2',
displaySubHandle: 'primary-zid-2',
profileImage: 'profile-image-url-2',
profileImage: 'blob://profile-image-url-2',
},
'matrix-id-3': {
userId: 'user-3',
Expand All @@ -404,17 +425,14 @@ describe('channels list saga', () => {
lastName: 'last-3',
primaryZID: '',
displaySubHandle: '',
profileImage: 'profile-image-url-3',
profileImage: 'blob://profile-image-url-3',
},
};

const clonedRooms = cloneDeep(rooms);
await expectSaga(mapToZeroUsers, clonedRooms)
await expectSaga(mapToZeroUsers, rooms)
.withReducer(rootReducer)
.provide([
[call(getZEROUsers, ['matrix-id-1', 'matrix-id-2', 'matrix-id-3']), zeroUsers],
])
.call(mapChannelMembers, clonedRooms, expectedMap)
.provide(getProviders(zeroUsers))
.call(mapChannelMembers, rooms, expectedMap)
.run();
});

Expand All @@ -423,12 +441,7 @@ describe('channels list saga', () => {

await expectSaga(mapToZeroUsers, rooms)
.withReducer(rootReducer, initialState.build())
.provide([
[call(getZEROUsers, ['matrix-id-1', 'matrix-id-2', 'matrix-id-3']), zeroUsers],
[call(downloadFile, 'profile-image-url-1'), 'profile-image-local-url-1'],
[call(downloadFile, 'profile-image-url-2'), 'profile-image-local-url-2'],
[call(downloadFile, 'profile-image-url-3'), 'profile-image-local-url-3'],
])
.provide(getProviders(zeroUsers))
.run();

expect(rooms[0].memberHistory).toIncludeSameMembers([
Expand All @@ -438,7 +451,7 @@ describe('channels list saga', () => {
profileId: 'profile-1',
firstName: 'first-1',
lastName: 'last-1',
profileImage: 'profile-image-local-url-1',
profileImage: 'blob://profile-image-url-1',
primaryZID: 'primary-zid-1',
displaySubHandle: 'primary-zid-1',
},
Expand All @@ -448,7 +461,7 @@ describe('channels list saga', () => {
profileId: 'profile-2',
firstName: 'first-2',
lastName: 'last-2',
profileImage: 'profile-image-local-url-2',
profileImage: 'blob://profile-image-url-2',
primaryZID: 'primary-zid-2',
displaySubHandle: 'primary-zid-2',
},
Expand All @@ -461,7 +474,7 @@ describe('channels list saga', () => {
profileId: 'profile-3',
firstName: 'first-3',
lastName: 'last-3',
profileImage: 'profile-image-local-url-3',
profileImage: 'blob://profile-image-url-3',
primaryZID: '',
displaySubHandle: '',
},
Expand All @@ -479,20 +492,15 @@ describe('channels list saga', () => {

await expectSaga(mapToZeroUsers, rooms)
.withReducer(rootReducer, initialState.build())
.provide([
[call(getZEROUsers, ['matrix-id-1', 'matrix-id-2', 'matrix-id-3']), zeroUsers],
[call(downloadFile, 'profile-image-url-1'), 'profile-image-local-url-1'],
[call(downloadFile, 'profile-image-url-2'), 'profile-image-local-url-2'],
[call(downloadFile, 'profile-image-url-3'), 'profile-image-local-url-3'],
])
.provide(getProviders(zeroUsers))
.run();

expect(rooms[0].messages[0].sender).toStrictEqual({
userId: 'user-1',
profileId: 'profile-1',
firstName: 'first-1',
lastName: 'last-1',
profileImage: 'profile-image-local-url-1',
profileImage: 'blob://profile-image-url-1',
primaryZID: 'primary-zid-1',
displaySubHandle: 'primary-zid-1',
});
Expand All @@ -501,7 +509,7 @@ describe('channels list saga', () => {
profileId: 'profile-2',
firstName: 'first-2',
lastName: 'last-2',
profileImage: 'profile-image-local-url-2',
profileImage: 'blob://profile-image-url-2',
primaryZID: 'primary-zid-2',
displaySubHandle: 'primary-zid-2',
});
Expand All @@ -510,7 +518,7 @@ describe('channels list saga', () => {
profileId: 'profile-3',
firstName: 'first-3',
lastName: 'last-3',
profileImage: 'profile-image-local-url-3',
profileImage: 'blob://profile-image-url-3',
primaryZID: '',
displaySubHandle: '',
});
Expand All @@ -524,7 +532,7 @@ describe('channels list saga', () => {

await expectSaga(mapToZeroUsers, rooms)
.withReducer(rootReducer, initialState.build())
.provide([[call(getZEROUsers, ['matrix-id-1', 'matrix-id-2', 'matrix-id-3']), zeroUsers]])
.provide(getProviders(zeroUsers))
.run();

expect(rooms[0].moderatorIds).toIncludeSameMembers(['user-1', 'user-2']);
Expand Down
Loading

0 comments on commit e1a40f1

Please sign in to comment.