From f556518fa1300b91bb5b035e463d80ff8556ce55 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Mon, 11 Sep 2023 21:24:29 -0300 Subject: [PATCH] fix: Consider only sessions from last few days for SAU/MAU (#30354) --- .changeset/strong-laws-pump.md | 8 + .../app/statistics/server/lib/SAUMonitor.ts | 38 +--- apps/meteor/server/models/raw/Sessions.ts | 205 +++++++++--------- .../src/models/ISessionsModel.ts | 4 + 4 files changed, 129 insertions(+), 126 deletions(-) create mode 100644 .changeset/strong-laws-pump.md diff --git a/.changeset/strong-laws-pump.md b/.changeset/strong-laws-pump.md new file mode 100644 index 000000000000..a4afefd65316 --- /dev/null +++ b/.changeset/strong-laws-pump.md @@ -0,0 +1,8 @@ +--- +'@rocket.chat/model-typings': patch +'@rocket.chat/meteor': patch +--- + +Change SAU aggregation to consider only sessions from few days ago instead of the whole past. + +This is particularly important for large workspaces in case the cron job did not run for some time, in that case the amount of sessions would accumulate and the aggregation would take a long time to run. diff --git a/apps/meteor/app/statistics/server/lib/SAUMonitor.ts b/apps/meteor/app/statistics/server/lib/SAUMonitor.ts index 79ce688cffd3..b3aa68337106 100644 --- a/apps/meteor/app/statistics/server/lib/SAUMonitor.ts +++ b/apps/meteor/app/statistics/server/lib/SAUMonitor.ts @@ -318,33 +318,19 @@ export class SAUMonitorClass { return; } - logger.info('[aggregate] - Aggregating data.'); - - const date = new Date(); - date.setDate(date.getDate() - 0); // yesterday - const yesterday = getDateObj(date); - - for await (const record of aggregates.dailySessionsOfYesterday(Sessions.col, yesterday)) { - await Sessions.updateOne( - { _id: `${record.userId}-${record.year}-${record.month}-${record.day}` }, - { $set: record }, - { upsert: true }, - ); + const today = new Date(); + + // get sessions from 3 days ago to make sure even if a few cron jobs were skipped, we still have the data + const threeDaysAgo = new Date(today.getFullYear(), today.getMonth(), today.getDate() - 3, 0, 0, 0, 0); + + const period = { start: getDateObj(threeDaysAgo), end: getDateObj(today) }; + + logger.info({ msg: '[aggregate] - Aggregating data.', period }); + + for await (const record of aggregates.dailySessions(Sessions.col, period)) { + await Sessions.updateDailySessionById(`${record.userId}-${record.year}-${record.month}-${record.day}`, record); } - await Sessions.updateMany( - { - type: 'session', - year: { $lte: yesterday.year }, - month: { $lte: yesterday.month }, - day: { $lte: yesterday.day }, - }, - { - $set: { - type: 'computed-session', - _computedAt: new Date(), - }, - }, - ); + await Sessions.updateAllSessionsByDateToComputed(period); } } diff --git a/apps/meteor/server/models/raw/Sessions.ts b/apps/meteor/server/models/raw/Sessions.ts index 68d5149232ed..c02fc8b5de99 100644 --- a/apps/meteor/server/models/raw/Sessions.ts +++ b/apps/meteor/server/models/raw/Sessions.ts @@ -167,9 +167,9 @@ const getProjectionByFullDate = (): { day: string; month: string; year: string } }); export const aggregates = { - dailySessionsOfYesterday( + dailySessions( collection: Collection, - { year, month, day }: DestructuredDate, + { start, end }: DestructuredRange, ): AggregationCursor< Pick & { time: number; @@ -178,115 +178,101 @@ export const aggregates = { _computedAt: string; } > { - return collection.aggregate< - Pick & { - time: number; - sessions: number; - devices: ISession['device'][]; - _computedAt: string; - } - >( - [ - { - $match: { - userId: { $exists: true }, - lastActivityAt: { $exists: true }, - device: { $exists: true }, - type: 'session', - $or: [ - { - year: { $lt: year }, - }, - { - year, - month: { $lt: month }, - }, - { - year, - month, - day: { $lte: day }, - }, - ], - }, - }, - { - $project: { - userId: 1, - device: 1, - day: 1, - month: 1, - year: 1, - mostImportantRole: 1, - time: { $trunc: { $divide: [{ $subtract: ['$lastActivityAt', '$loginAt'] }, 1000] } }, - }, - }, - { - $match: { - time: { $gt: 0 }, - }, + const pipeline = [ + { + $match: { + userId: { $exists: true }, + lastActivityAt: { $exists: true }, + device: { $exists: true }, + type: 'session', + ...matchBasedOnDate(start, end), }, - { - $group: { - _id: { - userId: '$userId', - device: '$device', - day: '$day', - month: '$month', - year: '$year', - }, - mostImportantRole: { $first: '$mostImportantRole' }, - time: { $sum: '$time' }, - sessions: { $sum: 1 }, - }, + }, + { + $project: { + userId: 1, + device: 1, + day: 1, + month: 1, + year: 1, + mostImportantRole: 1, + time: { $trunc: { $divide: [{ $subtract: ['$lastActivityAt', '$loginAt'] }, 1000] } }, }, - { - $sort: { - time: -1, - }, + }, + { + $match: { + time: { $gt: 0 }, }, - { - $group: { - _id: { - userId: '$_id.userId', - day: '$_id.day', - month: '$_id.month', - year: '$_id.year', - }, - mostImportantRole: { $first: '$mostImportantRole' }, - time: { $sum: '$time' }, - sessions: { $sum: '$sessions' }, - devices: { - $push: { - sessions: '$sessions', - time: '$time', - device: '$_id.device', - }, - }, + }, + { + $group: { + _id: { + userId: '$userId', + device: '$device', + day: '$day', + month: '$month', + year: '$year', }, + mostImportantRole: { $first: '$mostImportantRole' }, + time: { $sum: '$time' }, + sessions: { $sum: 1 }, }, - { - $sort: { - _id: 1, - }, + }, + { + $sort: { + time: -1, }, - { - $project: { - _id: 0, - type: { $literal: 'user_daily' }, - _computedAt: { $literal: new Date() }, + }, + { + $group: { + _id: { + userId: '$_id.userId', day: '$_id.day', month: '$_id.month', year: '$_id.year', - userId: '$_id.userId', - mostImportantRole: 1, - time: 1, - sessions: 1, - devices: 1, + }, + mostImportantRole: { $first: '$mostImportantRole' }, + time: { $sum: '$time' }, + sessions: { $sum: '$sessions' }, + devices: { + $push: { + sessions: '$sessions', + time: '$time', + device: '$_id.device', + }, }, }, - ], - { allowDiskUse: true }, - ); + }, + { + $sort: { + _id: 1, + }, + }, + { + $project: { + _id: 0, + type: { $literal: 'user_daily' }, + _computedAt: { $literal: new Date() }, + day: '$_id.day', + month: '$_id.month', + year: '$_id.year', + userId: '$_id.userId', + mostImportantRole: 1, + time: 1, + sessions: 1, + devices: 1, + }, + }, + ]; + + return collection.aggregate< + Pick & { + time: number; + sessions: number; + devices: ISession['device'][]; + _computedAt: string; + } + >(pipeline, { allowDiskUse: true }); }, async getUniqueUsersOfYesterday( @@ -1616,4 +1602,23 @@ export class SessionsRaw extends BaseRaw implements ISessionsModel { return this.col.bulkWrite(ops, { ordered: false }); } + + async updateDailySessionById(_id: ISession['_id'], record: Partial): Promise { + return this.updateOne({ _id }, { $set: record }, { upsert: true }); + } + + async updateAllSessionsByDateToComputed({ start, end }: DestructuredRange): Promise { + return this.updateMany( + { + type: 'session', + ...matchBasedOnDate(start, end), + }, + { + $set: { + type: 'computed-session', + _computedAt: new Date(), + }, + }, + ); + } } diff --git a/packages/model-typings/src/models/ISessionsModel.ts b/packages/model-typings/src/models/ISessionsModel.ts index cebe0c861d3f..1e6a36fd6f78 100644 --- a/packages/model-typings/src/models/ISessionsModel.ts +++ b/packages/model-typings/src/models/ISessionsModel.ts @@ -145,4 +145,8 @@ export interface ISessionsModel extends IBaseModel { }): Promise; createBatch(sessions: OptionalId[]): Promise; + + updateDailySessionById(_id: ISession['_id'], record: Partial): Promise; + + updateAllSessionsByDateToComputed({ start, end }: DestructuredRange): Promise; }