From 2b9f89aca653484b9e13d6eeff17abbc1bb50f83 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 27 Nov 2024 17:41:34 -0500 Subject: [PATCH 01/16] adding bounce tracking --- app/analytics/__tests__/collect.test.ts | 75 +++++++++++++++++++++++++ app/analytics/collect.ts | 52 +++++++++++++++-- 2 files changed, 122 insertions(+), 5 deletions(-) diff --git a/app/analytics/__tests__/collect.test.ts b/app/analytics/__tests__/collect.test.ts index 2bb47d9a..da97c685 100644 --- a/app/analytics/__tests__/collect.test.ts +++ b/app/analytics/__tests__/collect.test.ts @@ -70,6 +70,7 @@ describe("collectRequestHandler", () => { doubles: [ 1, // new visitor 1, // new session + 1, // new visit, so bounce ], indexes: [ "example", // site id is index @@ -95,6 +96,7 @@ describe("collectRequestHandler", () => { [ 1, // new visitor 1, // new session + 1, // new visit, so bounce ], ); }); @@ -123,6 +125,7 @@ describe("collectRequestHandler", () => { [ 0, // NOT a new visitor 0, // NOT a new session + 0, // NOT first or second visit ], ); }); @@ -157,6 +160,7 @@ describe("collectRequestHandler", () => { 1, // new visitor because a new day began 0, // NOT a new session because continuation of earlier session (< 30 mins) // (session logic doesn't care if a new day began or not) + 1, // new visitor so bounce counted ], ); }); @@ -185,6 +189,7 @@ describe("collectRequestHandler", () => { [ 1, // new visitor because > 30 days passed 1, // new session because > 30 minutes passed + 1, // new visitor so bounce ], ); }); @@ -213,6 +218,76 @@ describe("collectRequestHandler", () => { [ 1, // new visitor because > 24 hours passed 1, // new session because > 30 minutes passed + 1, // new visitor so bounce + ], + ); + }); + + test("if-modified-since is one second after midnight", () => { + const env = { + WEB_COUNTER_AE: { + writeDataPoint: vi.fn(), + } as AnalyticsEngineDataset, + } as Env; + + // midnight + const midnight = new Date(Math.floor(Date.now() / 8.64e7) * 8.64e7); + // set system time to midnight to see how the bounce works at the margin + vi.setSystemTime(midnight.getTime()); + // increment to one second after midnight + midnight.setSeconds(midnight.getSeconds() + 1); + + const request = httpMocks.createRequest( + // @ts-expect-error - we're mocking the request object + generateRequestParams({ + "if-modified-since": midnight.toUTCString(), + }), + ); + + collectRequestHandler(request as any, env); + + const writeDataPoint = env.WEB_COUNTER_AE.writeDataPoint; + expect((writeDataPoint as Mock).mock.calls[0][0]).toHaveProperty( + "doubles", + [ + 0, // new visitor because > 24 hours passed + 0, // new session because > 30 minutes passed + -1, // new visitor so bounce + ], + ); + }); + + test("if-modified-since is two seconds after midnight", () => { + const env = { + WEB_COUNTER_AE: { + writeDataPoint: vi.fn(), + } as AnalyticsEngineDataset, + } as Env; + + // midnight + const midnight = new Date(Math.floor(Date.now() / 8.64e7) * 8.64e7); + // set system to one second after midnight + midnight.setSeconds(midnight.getSeconds() + 1); + vi.setSystemTime(midnight.getTime()); + // increment to two seconds after midnight + midnight.setSeconds(midnight.getSeconds() + 1); + + const request = httpMocks.createRequest( + // @ts-expect-error - we're mocking the request object + generateRequestParams({ + "if-modified-since": midnight.toUTCString(), + }), + ); + + collectRequestHandler(request as any, env); + + const writeDataPoint = env.WEB_COUNTER_AE.writeDataPoint; + expect((writeDataPoint as Mock).mock.calls[0][0]).toHaveProperty( + "doubles", + [ + 0, // new visitor because > 24 hours passed + 0, // new session because > 30 minutes passed + 0, // new visitor so bounce ], ); }); diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index 76829976..069587fb 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -5,6 +5,44 @@ import type { RequestInit } from "@cloudflare/workers-types"; // Cookieless visitor/session tracking // Uses the approach described here: https://notes.normally.com/cookieless-unique-visitor-counts/ +function getMidnightDate(): Date { + const now = Date.now(); + // number of milliseconds in a day + const day = 8.64e7; + + return new Date(Math.floor(now / day) * day); +} + +function getNextModifiedDate(current: Date | null): Date { + const midnight = getMidnightDate(); + + // check if new day, if it is then set to midnight + let next = current ? current : midnight; + next = midnight.getTime() - next.getTime() > 0 ? midnight : next; + + // increment counter + next.setSeconds(next.getSeconds() + 1); + return next; +} + +function getBounce(current: Date | null): number { + if (!current) { + return 0; + } + + const midnight = getMidnightDate(); + const visits = (current.getTime() - midnight.getTime()) / 1000 - 1; + + switch (visits) { + case 0: + return 1; + case 1: + return -1; + default: + return 0; + } +} + function checkVisitorSession(ifModifiedSince: string | null): { newVisitor: boolean; newSession: boolean; @@ -62,9 +100,11 @@ export function collectRequestHandler(request: Request, env: Env) { parsedUserAgent.getBrowser().name; - const { newVisitor, newSession } = checkVisitorSession( - request.headers.get("if-modified-since"), - ); + const ifModifiedSince = request.headers.get("if-modified-since"); + const { newVisitor, newSession } = checkVisitorSession(ifModifiedSince); + const modifiedDate = ifModifiedSince + ? getNextModifiedDate(new Date(ifModifiedSince)) + : getNextModifiedDate(null); const data: DataPoint = { siteId: params.sid, @@ -73,6 +113,7 @@ export function collectRequestHandler(request: Request, env: Env) { referrer: params.r, newVisitor: newVisitor ? 1 : 0, newSession: newSession ? 1 : 0, + bounce: newVisitor ? 1 : getBounce(modifiedDate), // user agent stuff userAgent: userAgent, browserName: parsedUserAgent.getBrowser().name, @@ -104,7 +145,7 @@ export function collectRequestHandler(request: Request, env: Env) { Expires: "Mon, 01 Jan 1990 00:00:00 GMT", "Cache-Control": "no-cache", Pragma: "no-cache", - "Last-Modified": new Date().toUTCString(), + "Last-Modified": modifiedDate.toUTCString(), Tk: "N", // not tracking }, status: 200, @@ -127,6 +168,7 @@ interface DataPoint { // doubles newVisitor: number; newSession: number; + bounce: number; } // NOTE: Cloudflare Analytics Engine has limits on total number of bytes, number of fields, etc. @@ -148,7 +190,7 @@ export function writeDataPoint( data.deviceModel || "", // blob7 data.siteId || "", // blob8 ], - doubles: [data.newVisitor || 0, data.newSession || 0], + doubles: [data.newVisitor || 0, data.newSession || 0, data.bounce], }; if (!analyticsEngine) { From d54b298d2da44b926d8626ed5dcb0f55fe3b0b17 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 27 Nov 2024 18:31:40 -0500 Subject: [PATCH 02/16] query updates --- app/analytics/__tests__/query.test.ts | 8 +++++++- app/analytics/query.ts | 22 +++++++++++++++++----- app/analytics/schema.ts | 3 +++ app/routes/resources.stats.tsx | 9 ++++++++- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index fc66eea1..b9c56c36 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -195,16 +195,19 @@ describe("AnalyticsEngineAPI", () => { count: 3, isVisit: 1, isVisitor: 0, + bounce: 1, }, { count: 2, isVisit: 0, isVisitor: 0, + bounce: 0, }, { count: 1, isVisit: 0, isVisitor: 1, + bounce: -1, }, ], }), @@ -218,6 +221,7 @@ describe("AnalyticsEngineAPI", () => { views: 6, visits: 3, visitors: 1, + bounces: 2, }); }); }); @@ -325,9 +329,10 @@ describe("AnalyticsEngineAPI", () => { "SELECT blob4, " + "double1 as isVisitor, " + "double2 as isVisit, " + + "double3 as bounce, " + "SUM(_sample_interval) as count " + "FROM metricsDataset WHERE timestamp >= NOW() - INTERVAL '7' DAY AND timestamp < NOW() AND blob8 = 'example.com' AND blob4 = 'CA' " + - "GROUP BY blob4, double1, double2 " + + "GROUP BY blob4, double1, double2, double3 " + "ORDER BY count DESC LIMIT 10", ); expect(await result).toEqual({ @@ -335,6 +340,7 @@ describe("AnalyticsEngineAPI", () => { views: 3, visitors: 0, visits: 0, + bounces: 0, }, }); }); diff --git a/app/analytics/query.ts b/app/analytics/query.ts index 9e8079b5..81e6d69c 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -21,6 +21,7 @@ interface AnalyticsCountResult { views: number; visits: number; visitors: number; + bounces: number; } /** Given an AnalyticsCountResult object, and an object representing a row returned from @@ -33,6 +34,7 @@ function accumulateCountsFromRowResult( count: number; isVisitor: number; isVisit: number; + bounce: number; }, ) { if (row.isVisit == 1) { @@ -41,6 +43,10 @@ function accumulateCountsFromRowResult( if (row.isVisitor == 1) { counts.visitors += Number(row.count); } + if (row.bounce && row.bounce != 0) { + // bounce is either 1 or -1 + counts.bounces += Number(row.count) * row.bounce; + } counts.views += Number(row.count); } @@ -230,7 +236,7 @@ export class AnalyticsEngineAPI { /* output as UTC */ toDateTime(_bucket, 'Etc/UTC') as bucket FROM metricsDataset - WHERE timestamp >= toDateTime('${localStartTime.format("YYYY-MM-DD HH:mm:ss")}') + WHERE timestamp >= toDateTime('${localStartTime.format("YYYY-MM-DD HH:mm:ss")}') AND timestamp < toDateTime('${localEndTime.format("YYYY-MM-DD HH:mm:ss")}') AND ${ColumnMappings.siteId} = '${siteId}' ${filterStr} @@ -303,18 +309,20 @@ export class AnalyticsEngineAPI { const query = ` SELECT SUM(_sample_interval) as count, ${ColumnMappings.newVisitor} as isVisitor, - ${ColumnMappings.newSession} as isVisit + ${ColumnMappings.newSession} as isVisit, + ${ColumnMappings.bounce} as bounce FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} ${filterStr} AND ${siteIdColumn} = '${siteId}' - GROUP BY isVisitor, isVisit - ORDER BY isVisitor, isVisit ASC`; + GROUP BY isVisitor, isVisit, bounce + ORDER BY isVisitor, isVisit, bounce ASC`; type SelectionSet = { count: number; isVisitor: number; isVisit: number; + bounce: number; }; const queryResult = this.query(query); @@ -335,6 +343,7 @@ export class AnalyticsEngineAPI { views: 0, visitors: 0, visits: 0, + bounces: 0, }; // NOTE: note it's possible to get no results, or half results (i.e. a row where isVisit=1 but @@ -437,12 +446,13 @@ export class AnalyticsEngineAPI { SELECT ${_column}, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, + ${ColumnMappings.bounce} as bounce, SUM(_sample_interval) as count FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} AND ${ColumnMappings.siteId} = '${siteId}' ${filterStr} - GROUP BY ${_column}, ${ColumnMappings.newVisitor}, ${ColumnMappings.newSession} + GROUP BY ${_column}, ${ColumnMappings.newVisitor}, ${ColumnMappings.newSession}, ${ColumnMappings.bounce} ORDER BY count DESC LIMIT ${limit * page}`; @@ -450,6 +460,7 @@ export class AnalyticsEngineAPI { readonly count: number; readonly isVisitor: number; readonly isVisit: number; + readonly bounce: number; } & Record< (typeof ColumnMappings)[T], ColumnMappingToType<(typeof ColumnMappings)[T]> @@ -483,6 +494,7 @@ export class AnalyticsEngineAPI { views: 0, visitors: 0, visits: 0, + bounces: 0, } as AnalyticsCountResult; } diff --git a/app/analytics/schema.ts b/app/analytics/schema.ts index 4beb097c..a943d080 100644 --- a/app/analytics/schema.ts +++ b/app/analytics/schema.ts @@ -32,4 +32,7 @@ export const ColumnMappings = { // this record is a new session (resets after 30m inactivity) newSession: "double2", + + // this record is the bounce value + bounce: "double3", } as const; diff --git a/app/routes/resources.stats.tsx b/app/routes/resources.stats.tsx index 39492ba4..5ade6bcc 100644 --- a/app/routes/resources.stats.tsx +++ b/app/routes/resources.stats.tsx @@ -19,6 +19,7 @@ export async function loader({ context, request }: LoaderFunctionArgs) { views: counts.views, visits: counts.visits, visitors: counts.visitors, + bounces: counts.bounces, }); } @@ -35,7 +36,7 @@ export const StatsCard = ({ }) => { const dataFetcher = useFetcher(); - const { views, visits, visitors } = dataFetcher.data || {}; + const { views, visits, visitors, bounces } = dataFetcher.data || {}; const countFormatter = Intl.NumberFormat("en", { notation: "compact" }); useEffect(() => { @@ -76,6 +77,12 @@ export const StatsCard = ({ {visitors ? countFormatter.format(visitors) : "-"} +
+
Bounces
+
+ {bounces ? countFormatter.format(bounces) : "-"} +
+
From a2bd2c3d62648395c77aa3f72c083616fb40b949 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 27 Nov 2024 18:56:19 -0500 Subject: [PATCH 03/16] comment updates --- app/analytics/__tests__/collect.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/analytics/__tests__/collect.test.ts b/app/analytics/__tests__/collect.test.ts index da97c685..dcb0f87a 100644 --- a/app/analytics/__tests__/collect.test.ts +++ b/app/analytics/__tests__/collect.test.ts @@ -250,9 +250,9 @@ describe("collectRequestHandler", () => { expect((writeDataPoint as Mock).mock.calls[0][0]).toHaveProperty( "doubles", [ - 0, // new visitor because > 24 hours passed - 0, // new session because > 30 minutes passed - -1, // new visitor so bounce + 0, // NOT a new visitor + 0, // NOT a new session + -1, // First visit after the initial visit so decrement bounce ], ); }); @@ -285,9 +285,9 @@ describe("collectRequestHandler", () => { expect((writeDataPoint as Mock).mock.calls[0][0]).toHaveProperty( "doubles", [ - 0, // new visitor because > 24 hours passed - 0, // new session because > 30 minutes passed - 0, // new visitor so bounce + 0, // NOT a new visitor + 0, // NOT a new session + 0, // After the second visit so no bounce ], ); }); From e5af221ea3b7e09c8efeb826540434a3405bf5a8 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 27 Nov 2024 21:30:51 -0500 Subject: [PATCH 04/16] adding suggested fixes --- app/analytics/__tests__/collect.test.ts | 35 +++++++++++++++---------- app/analytics/collect.ts | 14 +++++----- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/app/analytics/__tests__/collect.test.ts b/app/analytics/__tests__/collect.test.ts index dcb0f87a..0a8a2fec 100644 --- a/app/analytics/__tests__/collect.test.ts +++ b/app/analytics/__tests__/collect.test.ts @@ -230,17 +230,20 @@ describe("collectRequestHandler", () => { } as AnalyticsEngineDataset, } as Env; - // midnight - const midnight = new Date(Math.floor(Date.now() / 8.64e7) * 8.64e7); - // set system time to midnight to see how the bounce works at the margin + const midnight = new Date(); + midnight.setHours(0, 0, 0, 0); + vi.setSystemTime(midnight.getTime()); - // increment to one second after midnight - midnight.setSeconds(midnight.getSeconds() + 1); + + const midnightPlusOneSecond = new Date(midnight.getTime()); + midnightPlusOneSecond.setSeconds( + midnightPlusOneSecond.getSeconds() + 1, + ); const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": midnight.toUTCString(), + "if-modified-since": midnightPlusOneSecond.toUTCString(), }), ); @@ -264,18 +267,22 @@ describe("collectRequestHandler", () => { } as AnalyticsEngineDataset, } as Env; - // midnight - const midnight = new Date(Math.floor(Date.now() / 8.64e7) * 8.64e7); - // set system to one second after midnight - midnight.setSeconds(midnight.getSeconds() + 1); - vi.setSystemTime(midnight.getTime()); - // increment to two seconds after midnight - midnight.setSeconds(midnight.getSeconds() + 1); + const midnightPlusOneSecond = new Date(); + midnightPlusOneSecond.setHours(0, 0, 1, 0); + + vi.setSystemTime(midnightPlusOneSecond.getTime()); + + const midnightPlusTwoSeconds = new Date( + midnightPlusOneSecond.getTime(), + ); + midnightPlusTwoSeconds.setSeconds( + midnightPlusTwoSeconds.getSeconds() + 1, + ); const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": midnight.toUTCString(), + "if-modified-since": midnightPlusTwoSeconds.toUTCString(), }), ); diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index 069587fb..9ec29733 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -6,11 +6,9 @@ import type { RequestInit } from "@cloudflare/workers-types"; // Uses the approach described here: https://notes.normally.com/cookieless-unique-visitor-counts/ function getMidnightDate(): Date { - const now = Date.now(); - // number of milliseconds in a day - const day = 8.64e7; - - return new Date(Math.floor(now / day) * day); + const midnight = new Date(); + midnight.setHours(0, 0, 0, 0); + return midnight; } function getNextModifiedDate(current: Date | null): Date { @@ -102,9 +100,9 @@ export function collectRequestHandler(request: Request, env: Env) { const ifModifiedSince = request.headers.get("if-modified-since"); const { newVisitor, newSession } = checkVisitorSession(ifModifiedSince); - const modifiedDate = ifModifiedSince - ? getNextModifiedDate(new Date(ifModifiedSince)) - : getNextModifiedDate(null); + const modifiedDate = getNextModifiedDate( + ifModifiedSince ? new Date(ifModifiedSince) : null, + ); const data: DataPoint = { siteId: params.sid, From bdd613a77896c2c3889ecdf897c219df69e8e962 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 27 Nov 2024 21:44:39 -0500 Subject: [PATCH 05/16] adding suggested fixes --- app/analytics/collect.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index 9ec29733..445996e6 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -12,6 +12,10 @@ function getMidnightDate(): Date { } function getNextModifiedDate(current: Date | null): Date { + if (current && isNaN(current.getTime())) { + current = null; + } + const midnight = getMidnightDate(); // check if new day, if it is then set to midnight From 3f233c9ac43905507413a20198c28b6bf8e2e982 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 27 Nov 2024 21:51:13 -0500 Subject: [PATCH 06/16] comments --- app/analytics/collect.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index 445996e6..d296c75a 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -12,6 +12,7 @@ function getMidnightDate(): Date { } function getNextModifiedDate(current: Date | null): Date { + // in case date is an 'Invalid Date' if (current && isNaN(current.getTime())) { current = null; } From faf1002d15caa7591c30a1e171b58e964c2fdcd1 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Thu, 28 Nov 2024 08:54:54 -0500 Subject: [PATCH 07/16] updating implementation to rely on milliseconds for bounce --- app/analytics/__tests__/collect.test.ts | 47 +++++++++---------------- app/analytics/collect.ts | 43 +++++++++++----------- 2 files changed, 38 insertions(+), 52 deletions(-) diff --git a/app/analytics/__tests__/collect.test.ts b/app/analytics/__tests__/collect.test.ts index 0a8a2fec..7b67dd10 100644 --- a/app/analytics/__tests__/collect.test.ts +++ b/app/analytics/__tests__/collect.test.ts @@ -108,12 +108,15 @@ describe("collectRequestHandler", () => { } as AnalyticsEngineDataset, } as Env; + const fiveMinutes = new Date( + Date.now() - 5 * 60 * 1000, // 5 mins ago + ); + fiveMinutes.setMilliseconds(1); + const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": new Date( - Date.now() - 5 * 60 * 1000, // 5 mins ago - ).toUTCString(), + "if-modified-since": fiveMinutes.toISOString(), }), ); @@ -147,7 +150,7 @@ describe("collectRequestHandler", () => { generateRequestParams({ "if-modified-since": new Date( Date.now() - 25 * 60 * 1000, // 25 minutes ago - ).toUTCString(), + ).toISOString(), }), ); @@ -177,7 +180,7 @@ describe("collectRequestHandler", () => { generateRequestParams({ "if-modified-since": new Date( Date.now() - 31 * 24 * 60 * 60 * 1000, // 31 days ago - ).toUTCString(), + ).toISOString(), }), ); @@ -206,7 +209,7 @@ describe("collectRequestHandler", () => { generateRequestParams({ "if-modified-since": new Date( Date.now() - 24 * 60 * 60 * 1000, // 24 hours ago - ).toUTCString(), + ).toISOString(), }), ); @@ -223,27 +226,20 @@ describe("collectRequestHandler", () => { ); }); - test("if-modified-since is one second after midnight", () => { + test("if-modified-since has zero milliseconds", () => { const env = { WEB_COUNTER_AE: { writeDataPoint: vi.fn(), } as AnalyticsEngineDataset, } as Env; - const midnight = new Date(); - midnight.setHours(0, 0, 0, 0); - - vi.setSystemTime(midnight.getTime()); - - const midnightPlusOneSecond = new Date(midnight.getTime()); - midnightPlusOneSecond.setSeconds( - midnightPlusOneSecond.getSeconds() + 1, - ); + const now = new Date(); + now.setMilliseconds(0); const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": midnightPlusOneSecond.toUTCString(), + "if-modified-since": now.toISOString(), }), ); @@ -260,29 +256,20 @@ describe("collectRequestHandler", () => { ); }); - test("if-modified-since is two seconds after midnight", () => { + test("if-modified-since has one millisecond", () => { const env = { WEB_COUNTER_AE: { writeDataPoint: vi.fn(), } as AnalyticsEngineDataset, } as Env; - const midnightPlusOneSecond = new Date(); - midnightPlusOneSecond.setHours(0, 0, 1, 0); - - vi.setSystemTime(midnightPlusOneSecond.getTime()); - - const midnightPlusTwoSeconds = new Date( - midnightPlusOneSecond.getTime(), - ); - midnightPlusTwoSeconds.setSeconds( - midnightPlusTwoSeconds.getSeconds() + 1, - ); + const now = new Date(); + now.setMilliseconds(1); const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": midnightPlusTwoSeconds.toUTCString(), + "if-modified-since": now.toISOString(), }), ); diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index d296c75a..fd63d120 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -5,38 +5,37 @@ import type { RequestInit } from "@cloudflare/workers-types"; // Cookieless visitor/session tracking // Uses the approach described here: https://notes.normally.com/cookieless-unique-visitor-counts/ -function getMidnightDate(): Date { - const midnight = new Date(); - midnight.setHours(0, 0, 0, 0); - return midnight; -} - function getNextModifiedDate(current: Date | null): Date { // in case date is an 'Invalid Date' if (current && isNaN(current.getTime())) { current = null; } - const midnight = getMidnightDate(); - - // check if new day, if it is then set to midnight - let next = current ? current : midnight; - next = midnight.getTime() - next.getTime() > 0 ? midnight : next; + const now = new Date(); - // increment counter - next.setSeconds(next.getSeconds() + 1); - return next; -} - -function getBounce(current: Date | null): number { if (!current) { - return 0; + now.setMilliseconds(0); + return now; } - const midnight = getMidnightDate(); - const visits = (current.getTime() - midnight.getTime()) / 1000 - 1; + // update bounce + switch (current.getMilliseconds()) { + case 0: + now.setMilliseconds(1); + break; + case 1: + now.setMilliseconds(2); + break; + default: + now.setMilliseconds(3); + break; + } + + return now; +} - switch (visits) { +function getBounce(current: Date): number { + switch (current.getMilliseconds()) { case 0: return 1; case 1: @@ -148,7 +147,7 @@ export function collectRequestHandler(request: Request, env: Env) { Expires: "Mon, 01 Jan 1990 00:00:00 GMT", "Cache-Control": "no-cache", Pragma: "no-cache", - "Last-Modified": modifiedDate.toUTCString(), + "Last-Modified": modifiedDate.toISOString(), Tk: "N", // not tracking }, status: 200, From 920ea1c7b86df91d7bbeb39dfce3f6eeb4114d76 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Thu, 28 Nov 2024 09:22:03 -0500 Subject: [PATCH 08/16] var rename and improving millisecond set --- app/analytics/collect.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index fd63d120..f259f55f 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -5,33 +5,33 @@ import type { RequestInit } from "@cloudflare/workers-types"; // Cookieless visitor/session tracking // Uses the approach described here: https://notes.normally.com/cookieless-unique-visitor-counts/ -function getNextModifiedDate(current: Date | null): Date { +function getNextModifiedDate(current: Date | null, newVisit: boolean): Date { // in case date is an 'Invalid Date' if (current && isNaN(current.getTime())) { current = null; } - const now = new Date(); + const today = new Date(); - if (!current) { - now.setMilliseconds(0); - return now; + if (!current || newVisit) { + today.setMilliseconds(0); + return today; } // update bounce switch (current.getMilliseconds()) { case 0: - now.setMilliseconds(1); + today.setMilliseconds(1); break; case 1: - now.setMilliseconds(2); + today.setMilliseconds(2); break; default: - now.setMilliseconds(3); + today.setMilliseconds(3); break; } - return now; + return today; } function getBounce(current: Date): number { @@ -106,6 +106,7 @@ export function collectRequestHandler(request: Request, env: Env) { const { newVisitor, newSession } = checkVisitorSession(ifModifiedSince); const modifiedDate = getNextModifiedDate( ifModifiedSince ? new Date(ifModifiedSince) : null, + newVisitor, ); const data: DataPoint = { @@ -115,7 +116,7 @@ export function collectRequestHandler(request: Request, env: Env) { referrer: params.r, newVisitor: newVisitor ? 1 : 0, newSession: newSession ? 1 : 0, - bounce: newVisitor ? 1 : getBounce(modifiedDate), + bounce: getBounce(modifiedDate), // user agent stuff userAgent: userAgent, browserName: parsedUserAgent.getBrowser().name, From 0225c8a7d29d55e99e12d2877659abb0a88a569b Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Sat, 30 Nov 2024 14:21:35 -0500 Subject: [PATCH 09/16] suggested fixes --- app/analytics/__tests__/query.test.ts | 10 +++++----- app/analytics/collect.ts | 22 ++++++++++++++-------- app/analytics/query.ts | 21 ++++++++++----------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index b9c56c36..41121add 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -187,7 +187,7 @@ describe("AnalyticsEngineAPI", () => { }); describe("getCounts", () => { - test("should return an object with view, visit, and visitor counts", async () => { + test("should return an object with view, visit, visitor, and bounce counts", async () => { fetch.mockResolvedValue( createFetchResponse({ data: [ @@ -195,19 +195,19 @@ describe("AnalyticsEngineAPI", () => { count: 3, isVisit: 1, isVisitor: 0, - bounce: 1, + isBounce: 1, }, { count: 2, isVisit: 0, isVisitor: 0, - bounce: 0, + isBounce: 0, }, { count: 1, isVisit: 0, isVisitor: 1, - bounce: -1, + isBounce: -1, }, ], }), @@ -329,7 +329,7 @@ describe("AnalyticsEngineAPI", () => { "SELECT blob4, " + "double1 as isVisitor, " + "double2 as isVisit, " + - "double3 as bounce, " + + "double3 as isBounce, " + "SUM(_sample_interval) as count " + "FROM metricsDataset WHERE timestamp >= NOW() - INTERVAL '7' DAY AND timestamp < NOW() AND blob8 = 'example.com' AND blob4 = 'CA' " + "GROUP BY blob4, double1, double2, double3 " + diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index f259f55f..a15c4bb0 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -11,30 +11,36 @@ function getNextModifiedDate(current: Date | null, newVisit: boolean): Date { current = null; } - const today = new Date(); + const nextModifiedDate = new Date(); if (!current || newVisit) { - today.setMilliseconds(0); - return today; + nextModifiedDate.setMilliseconds(0); + return nextModifiedDate; } - // update bounce + // update bounce (tracked in milliseconds) + // 3 states of bounce: bounce (0), no bounce (1), done (2) switch (current.getMilliseconds()) { + // if was bounce move to no bounce case 0: - today.setMilliseconds(1); + nextModifiedDate.setMilliseconds(1); break; + // if was no bounce move to done case 1: - today.setMilliseconds(2); + nextModifiedDate.setMilliseconds(2); break; + // set value 3 to indicate done with bounce default: - today.setMilliseconds(3); + nextModifiedDate.setMilliseconds(3); break; } - return today; + return nextModifiedDate; } function getBounce(current: Date): number { + // get bounce value (tracked in milliseconds, see getNextModifiedDate) + // bounce (1), no bounce (-1), done (0) switch (current.getMilliseconds()) { case 0: return 1; diff --git a/app/analytics/query.ts b/app/analytics/query.ts index 81e6d69c..96d839d1 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -25,7 +25,7 @@ interface AnalyticsCountResult { } /** Given an AnalyticsCountResult object, and an object representing a row returned from - * CF Analytics Engine w/ counts grouped by isVisitor and isVisit, accumulate view, + * CF Analytics Engine w/ counts grouped by isVisitor, isVisit, and isBounce, accumulate view, * visit, and visitor counts. */ function accumulateCountsFromRowResult( @@ -34,7 +34,7 @@ function accumulateCountsFromRowResult( count: number; isVisitor: number; isVisit: number; - bounce: number; + isBounce: number; }, ) { if (row.isVisit == 1) { @@ -43,9 +43,8 @@ function accumulateCountsFromRowResult( if (row.isVisitor == 1) { counts.visitors += Number(row.count); } - if (row.bounce && row.bounce != 0) { - // bounce is either 1 or -1 - counts.bounces += Number(row.count) * row.bounce; + if (row.isBounce == 1 || row.isBounce == -1) { + counts.bounces += Number(row.count) * row.isBounce; } counts.views += Number(row.count); } @@ -310,19 +309,19 @@ export class AnalyticsEngineAPI { SELECT SUM(_sample_interval) as count, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, - ${ColumnMappings.bounce} as bounce + ${ColumnMappings.bounce} as isBounce FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} ${filterStr} AND ${siteIdColumn} = '${siteId}' - GROUP BY isVisitor, isVisit, bounce - ORDER BY isVisitor, isVisit, bounce ASC`; + GROUP BY isVisitor, isVisit, isBounce + ORDER BY isVisitor, isVisit, isBounce ASC`; type SelectionSet = { count: number; isVisitor: number; isVisit: number; - bounce: number; + isBounce: number; }; const queryResult = this.query(query); @@ -446,7 +445,7 @@ export class AnalyticsEngineAPI { SELECT ${_column}, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, - ${ColumnMappings.bounce} as bounce, + ${ColumnMappings.bounce} as isBounce, SUM(_sample_interval) as count FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} @@ -460,7 +459,7 @@ export class AnalyticsEngineAPI { readonly count: number; readonly isVisitor: number; readonly isVisit: number; - readonly bounce: number; + readonly isBounce: number; } & Record< (typeof ColumnMappings)[T], ColumnMappingToType<(typeof ColumnMappings)[T]> From 1da63bf709929851ff07ebcecf372d6c83ac6f02 Mon Sep 17 00:00:00 2001 From: Saad Makrod Date: Wed, 4 Dec 2024 18:35:39 -0500 Subject: [PATCH 10/16] reverting to initial implementation --- app/analytics/__tests__/collect.test.ts | 47 +++++++++++++-------- app/analytics/__tests__/query.test.ts | 10 ++--- app/analytics/collect.ts | 54 +++++++++++-------------- app/analytics/query.ts | 21 +++++----- 4 files changed, 70 insertions(+), 62 deletions(-) diff --git a/app/analytics/__tests__/collect.test.ts b/app/analytics/__tests__/collect.test.ts index 7b67dd10..0a8a2fec 100644 --- a/app/analytics/__tests__/collect.test.ts +++ b/app/analytics/__tests__/collect.test.ts @@ -108,15 +108,12 @@ describe("collectRequestHandler", () => { } as AnalyticsEngineDataset, } as Env; - const fiveMinutes = new Date( - Date.now() - 5 * 60 * 1000, // 5 mins ago - ); - fiveMinutes.setMilliseconds(1); - const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": fiveMinutes.toISOString(), + "if-modified-since": new Date( + Date.now() - 5 * 60 * 1000, // 5 mins ago + ).toUTCString(), }), ); @@ -150,7 +147,7 @@ describe("collectRequestHandler", () => { generateRequestParams({ "if-modified-since": new Date( Date.now() - 25 * 60 * 1000, // 25 minutes ago - ).toISOString(), + ).toUTCString(), }), ); @@ -180,7 +177,7 @@ describe("collectRequestHandler", () => { generateRequestParams({ "if-modified-since": new Date( Date.now() - 31 * 24 * 60 * 60 * 1000, // 31 days ago - ).toISOString(), + ).toUTCString(), }), ); @@ -209,7 +206,7 @@ describe("collectRequestHandler", () => { generateRequestParams({ "if-modified-since": new Date( Date.now() - 24 * 60 * 60 * 1000, // 24 hours ago - ).toISOString(), + ).toUTCString(), }), ); @@ -226,20 +223,27 @@ describe("collectRequestHandler", () => { ); }); - test("if-modified-since has zero milliseconds", () => { + test("if-modified-since is one second after midnight", () => { const env = { WEB_COUNTER_AE: { writeDataPoint: vi.fn(), } as AnalyticsEngineDataset, } as Env; - const now = new Date(); - now.setMilliseconds(0); + const midnight = new Date(); + midnight.setHours(0, 0, 0, 0); + + vi.setSystemTime(midnight.getTime()); + + const midnightPlusOneSecond = new Date(midnight.getTime()); + midnightPlusOneSecond.setSeconds( + midnightPlusOneSecond.getSeconds() + 1, + ); const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": now.toISOString(), + "if-modified-since": midnightPlusOneSecond.toUTCString(), }), ); @@ -256,20 +260,29 @@ describe("collectRequestHandler", () => { ); }); - test("if-modified-since has one millisecond", () => { + test("if-modified-since is two seconds after midnight", () => { const env = { WEB_COUNTER_AE: { writeDataPoint: vi.fn(), } as AnalyticsEngineDataset, } as Env; - const now = new Date(); - now.setMilliseconds(1); + const midnightPlusOneSecond = new Date(); + midnightPlusOneSecond.setHours(0, 0, 1, 0); + + vi.setSystemTime(midnightPlusOneSecond.getTime()); + + const midnightPlusTwoSeconds = new Date( + midnightPlusOneSecond.getTime(), + ); + midnightPlusTwoSeconds.setSeconds( + midnightPlusTwoSeconds.getSeconds() + 1, + ); const request = httpMocks.createRequest( // @ts-expect-error - we're mocking the request object generateRequestParams({ - "if-modified-since": now.toISOString(), + "if-modified-since": midnightPlusTwoSeconds.toUTCString(), }), ); diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index 41121add..b9c56c36 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -187,7 +187,7 @@ describe("AnalyticsEngineAPI", () => { }); describe("getCounts", () => { - test("should return an object with view, visit, visitor, and bounce counts", async () => { + test("should return an object with view, visit, and visitor counts", async () => { fetch.mockResolvedValue( createFetchResponse({ data: [ @@ -195,19 +195,19 @@ describe("AnalyticsEngineAPI", () => { count: 3, isVisit: 1, isVisitor: 0, - isBounce: 1, + bounce: 1, }, { count: 2, isVisit: 0, isVisitor: 0, - isBounce: 0, + bounce: 0, }, { count: 1, isVisit: 0, isVisitor: 1, - isBounce: -1, + bounce: -1, }, ], }), @@ -329,7 +329,7 @@ describe("AnalyticsEngineAPI", () => { "SELECT blob4, " + "double1 as isVisitor, " + "double2 as isVisit, " + - "double3 as isBounce, " + + "double3 as bounce, " + "SUM(_sample_interval) as count " + "FROM metricsDataset WHERE timestamp >= NOW() - INTERVAL '7' DAY AND timestamp < NOW() AND blob8 = 'example.com' AND blob4 = 'CA' " + "GROUP BY blob4, double1, double2, double3 " + diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index a15c4bb0..d296c75a 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -5,43 +5,38 @@ import type { RequestInit } from "@cloudflare/workers-types"; // Cookieless visitor/session tracking // Uses the approach described here: https://notes.normally.com/cookieless-unique-visitor-counts/ -function getNextModifiedDate(current: Date | null, newVisit: boolean): Date { +function getMidnightDate(): Date { + const midnight = new Date(); + midnight.setHours(0, 0, 0, 0); + return midnight; +} + +function getNextModifiedDate(current: Date | null): Date { // in case date is an 'Invalid Date' if (current && isNaN(current.getTime())) { current = null; } - const nextModifiedDate = new Date(); + const midnight = getMidnightDate(); - if (!current || newVisit) { - nextModifiedDate.setMilliseconds(0); - return nextModifiedDate; - } + // check if new day, if it is then set to midnight + let next = current ? current : midnight; + next = midnight.getTime() - next.getTime() > 0 ? midnight : next; - // update bounce (tracked in milliseconds) - // 3 states of bounce: bounce (0), no bounce (1), done (2) - switch (current.getMilliseconds()) { - // if was bounce move to no bounce - case 0: - nextModifiedDate.setMilliseconds(1); - break; - // if was no bounce move to done - case 1: - nextModifiedDate.setMilliseconds(2); - break; - // set value 3 to indicate done with bounce - default: - nextModifiedDate.setMilliseconds(3); - break; + // increment counter + next.setSeconds(next.getSeconds() + 1); + return next; +} + +function getBounce(current: Date | null): number { + if (!current) { + return 0; } - return nextModifiedDate; -} + const midnight = getMidnightDate(); + const visits = (current.getTime() - midnight.getTime()) / 1000 - 1; -function getBounce(current: Date): number { - // get bounce value (tracked in milliseconds, see getNextModifiedDate) - // bounce (1), no bounce (-1), done (0) - switch (current.getMilliseconds()) { + switch (visits) { case 0: return 1; case 1: @@ -112,7 +107,6 @@ export function collectRequestHandler(request: Request, env: Env) { const { newVisitor, newSession } = checkVisitorSession(ifModifiedSince); const modifiedDate = getNextModifiedDate( ifModifiedSince ? new Date(ifModifiedSince) : null, - newVisitor, ); const data: DataPoint = { @@ -122,7 +116,7 @@ export function collectRequestHandler(request: Request, env: Env) { referrer: params.r, newVisitor: newVisitor ? 1 : 0, newSession: newSession ? 1 : 0, - bounce: getBounce(modifiedDate), + bounce: newVisitor ? 1 : getBounce(modifiedDate), // user agent stuff userAgent: userAgent, browserName: parsedUserAgent.getBrowser().name, @@ -154,7 +148,7 @@ export function collectRequestHandler(request: Request, env: Env) { Expires: "Mon, 01 Jan 1990 00:00:00 GMT", "Cache-Control": "no-cache", Pragma: "no-cache", - "Last-Modified": modifiedDate.toISOString(), + "Last-Modified": modifiedDate.toUTCString(), Tk: "N", // not tracking }, status: 200, diff --git a/app/analytics/query.ts b/app/analytics/query.ts index 96d839d1..81e6d69c 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -25,7 +25,7 @@ interface AnalyticsCountResult { } /** Given an AnalyticsCountResult object, and an object representing a row returned from - * CF Analytics Engine w/ counts grouped by isVisitor, isVisit, and isBounce, accumulate view, + * CF Analytics Engine w/ counts grouped by isVisitor and isVisit, accumulate view, * visit, and visitor counts. */ function accumulateCountsFromRowResult( @@ -34,7 +34,7 @@ function accumulateCountsFromRowResult( count: number; isVisitor: number; isVisit: number; - isBounce: number; + bounce: number; }, ) { if (row.isVisit == 1) { @@ -43,8 +43,9 @@ function accumulateCountsFromRowResult( if (row.isVisitor == 1) { counts.visitors += Number(row.count); } - if (row.isBounce == 1 || row.isBounce == -1) { - counts.bounces += Number(row.count) * row.isBounce; + if (row.bounce && row.bounce != 0) { + // bounce is either 1 or -1 + counts.bounces += Number(row.count) * row.bounce; } counts.views += Number(row.count); } @@ -309,19 +310,19 @@ export class AnalyticsEngineAPI { SELECT SUM(_sample_interval) as count, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, - ${ColumnMappings.bounce} as isBounce + ${ColumnMappings.bounce} as bounce FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} ${filterStr} AND ${siteIdColumn} = '${siteId}' - GROUP BY isVisitor, isVisit, isBounce - ORDER BY isVisitor, isVisit, isBounce ASC`; + GROUP BY isVisitor, isVisit, bounce + ORDER BY isVisitor, isVisit, bounce ASC`; type SelectionSet = { count: number; isVisitor: number; isVisit: number; - isBounce: number; + bounce: number; }; const queryResult = this.query(query); @@ -445,7 +446,7 @@ export class AnalyticsEngineAPI { SELECT ${_column}, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, - ${ColumnMappings.bounce} as isBounce, + ${ColumnMappings.bounce} as bounce, SUM(_sample_interval) as count FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} @@ -459,7 +460,7 @@ export class AnalyticsEngineAPI { readonly count: number; readonly isVisitor: number; readonly isVisit: number; - readonly isBounce: number; + readonly bounce: number; } & Record< (typeof ColumnMappings)[T], ColumnMappingToType<(typeof ColumnMappings)[T]> From d7817a25df0ed361715e663e458fc3a8cb07933b Mon Sep 17 00:00:00 2001 From: Ben Vinegar <2153+benvinegar@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:42:02 -0500 Subject: [PATCH 11/16] Rename query bounce column result to isBounce --- app/analytics/__tests__/query.test.ts | 8 ++++---- app/analytics/query.ts | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index b9c56c36..7018ebe9 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -195,19 +195,19 @@ describe("AnalyticsEngineAPI", () => { count: 3, isVisit: 1, isVisitor: 0, - bounce: 1, + isBounce: 1, }, { count: 2, isVisit: 0, isVisitor: 0, - bounce: 0, + isBounce: 0, }, { count: 1, isVisit: 0, isVisitor: 1, - bounce: -1, + isBounce: -1, }, ], }), @@ -329,7 +329,7 @@ describe("AnalyticsEngineAPI", () => { "SELECT blob4, " + "double1 as isVisitor, " + "double2 as isVisit, " + - "double3 as bounce, " + + "double3 as isBounce, " + "SUM(_sample_interval) as count " + "FROM metricsDataset WHERE timestamp >= NOW() - INTERVAL '7' DAY AND timestamp < NOW() AND blob8 = 'example.com' AND blob4 = 'CA' " + "GROUP BY blob4, double1, double2, double3 " + diff --git a/app/analytics/query.ts b/app/analytics/query.ts index 81e6d69c..71fbfe77 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -34,7 +34,7 @@ function accumulateCountsFromRowResult( count: number; isVisitor: number; isVisit: number; - bounce: number; + isBounce: number; }, ) { if (row.isVisit == 1) { @@ -43,9 +43,9 @@ function accumulateCountsFromRowResult( if (row.isVisitor == 1) { counts.visitors += Number(row.count); } - if (row.bounce && row.bounce != 0) { + if (row.isBounce && row.isBounce != 0) { // bounce is either 1 or -1 - counts.bounces += Number(row.count) * row.bounce; + counts.bounces += Number(row.count) * row.isBounce; } counts.views += Number(row.count); } @@ -310,19 +310,19 @@ export class AnalyticsEngineAPI { SELECT SUM(_sample_interval) as count, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, - ${ColumnMappings.bounce} as bounce + ${ColumnMappings.bounce} as isBounce FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} ${filterStr} AND ${siteIdColumn} = '${siteId}' - GROUP BY isVisitor, isVisit, bounce - ORDER BY isVisitor, isVisit, bounce ASC`; + GROUP BY isVisitor, isVisit, isBounce + ORDER BY isVisitor, isVisit, isBounce ASC`; type SelectionSet = { count: number; isVisitor: number; isVisit: number; - bounce: number; + isBounce: number; }; const queryResult = this.query(query); @@ -446,7 +446,7 @@ export class AnalyticsEngineAPI { SELECT ${_column}, ${ColumnMappings.newVisitor} as isVisitor, ${ColumnMappings.newSession} as isVisit, - ${ColumnMappings.bounce} as bounce, + ${ColumnMappings.bounce} as isBounce, SUM(_sample_interval) as count FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} @@ -460,7 +460,7 @@ export class AnalyticsEngineAPI { readonly count: number; readonly isVisitor: number; readonly isVisit: number; - readonly bounce: number; + readonly isBounce: number; } & Record< (typeof ColumnMappings)[T], ColumnMappingToType<(typeof ColumnMappings)[T]> From 3d648e2cc559a775a2eac57e200c392d7456809a Mon Sep 17 00:00:00 2001 From: Ben Vinegar <2153+benvinegar@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:58:17 -0500 Subject: [PATCH 12/16] Remove visits/sessions entirely --- app/analytics/__tests__/collect.test.ts | 17 +++++++------ app/analytics/__tests__/query.test.ts | 10 ++------ app/analytics/collect.ts | 18 +++----------- app/analytics/query.ts | 24 +++++-------------- app/routes/__tests__/dashboard.test.tsx | 3 --- app/routes/__tests__/resources.stats.test.tsx | 2 -- app/routes/resources.stats.tsx | 9 +------ 7 files changed, 20 insertions(+), 63 deletions(-) diff --git a/app/analytics/__tests__/collect.test.ts b/app/analytics/__tests__/collect.test.ts index 0a8a2fec..b89170c7 100644 --- a/app/analytics/__tests__/collect.test.ts +++ b/app/analytics/__tests__/collect.test.ts @@ -69,7 +69,7 @@ describe("collectRequestHandler", () => { ], doubles: [ 1, // new visitor - 1, // new session + 0, // DEAD COLUMN (was session) 1, // new visit, so bounce ], indexes: [ @@ -95,7 +95,7 @@ describe("collectRequestHandler", () => { "doubles", [ 1, // new visitor - 1, // new session + 0, // DEAD COLUMN (was session) 1, // new visit, so bounce ], ); @@ -124,7 +124,7 @@ describe("collectRequestHandler", () => { "doubles", [ 0, // NOT a new visitor - 0, // NOT a new session + 0, // DEAD COLUMN (was session) 0, // NOT first or second visit ], ); @@ -158,8 +158,7 @@ describe("collectRequestHandler", () => { "doubles", [ 1, // new visitor because a new day began - 0, // NOT a new session because continuation of earlier session (< 30 mins) - // (session logic doesn't care if a new day began or not) + 0, // DEAD COLUMN (was session) 1, // new visitor so bounce counted ], ); @@ -188,7 +187,7 @@ describe("collectRequestHandler", () => { "doubles", [ 1, // new visitor because > 30 days passed - 1, // new session because > 30 minutes passed + 0, // DEAD COLUMN (was session) 1, // new visitor so bounce ], ); @@ -217,7 +216,7 @@ describe("collectRequestHandler", () => { "doubles", [ 1, // new visitor because > 24 hours passed - 1, // new session because > 30 minutes passed + 0, // DEAD COLUMN (was session) 1, // new visitor so bounce ], ); @@ -254,7 +253,7 @@ describe("collectRequestHandler", () => { "doubles", [ 0, // NOT a new visitor - 0, // NOT a new session + 0, // DEAD COLUMN (was session) -1, // First visit after the initial visit so decrement bounce ], ); @@ -293,7 +292,7 @@ describe("collectRequestHandler", () => { "doubles", [ 0, // NOT a new visitor - 0, // NOT a new session + 0, // DEAD COLUMN (was session) 0, // After the second visit so no bounce ], ); diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index 7018ebe9..f078cfd5 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -187,25 +187,22 @@ describe("AnalyticsEngineAPI", () => { }); describe("getCounts", () => { - test("should return an object with view, visit, and visitor counts", async () => { + test("should return an object with view, visitor, and bounce counts", async () => { fetch.mockResolvedValue( createFetchResponse({ data: [ { count: 3, - isVisit: 1, isVisitor: 0, isBounce: 1, }, { count: 2, - isVisit: 0, isVisitor: 0, isBounce: 0, }, { count: 1, - isVisit: 0, isVisitor: 1, isBounce: -1, }, @@ -219,7 +216,6 @@ describe("AnalyticsEngineAPI", () => { expect(fetch).toHaveBeenCalled(); expect(await result).toEqual({ views: 6, - visits: 3, visitors: 1, bounces: 2, }); @@ -328,18 +324,16 @@ describe("AnalyticsEngineAPI", () => { ).toEqual( "SELECT blob4, " + "double1 as isVisitor, " + - "double2 as isVisit, " + "double3 as isBounce, " + "SUM(_sample_interval) as count " + "FROM metricsDataset WHERE timestamp >= NOW() - INTERVAL '7' DAY AND timestamp < NOW() AND blob8 = 'example.com' AND blob4 = 'CA' " + - "GROUP BY blob4, double1, double2, double3 " + + "GROUP BY blob4, double1, double3 " + "ORDER BY count DESC LIMIT 10", ); expect(await result).toEqual({ CA: { views: 3, visitors: 0, - visits: 0, bounces: 0, }, }); diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index d296c75a..efc745c1 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -48,12 +48,9 @@ function getBounce(current: Date | null): number { function checkVisitorSession(ifModifiedSince: string | null): { newVisitor: boolean; - newSession: boolean; } { let newVisitor = true; - let newSession = true; - const minutesUntilSessionResets = 30; if (ifModifiedSince) { // check today is a new day vs ifModifiedSince const today = new Date(); @@ -66,18 +63,9 @@ function checkVisitorSession(ifModifiedSince: string | null): { // if ifModifiedSince is today, this is not a new visitor newVisitor = false; } - - // check ifModifiedSince is less than 30 mins ago - if ( - Date.now() - new Date(ifModifiedSince).getTime() < - minutesUntilSessionResets * 60 * 1000 - ) { - // this is a continuation of the same session - newSession = false; - } } - return { newVisitor, newSession }; + return { newVisitor }; } function extractParamsFromQueryString(requestUrl: string): { @@ -104,7 +92,7 @@ export function collectRequestHandler(request: Request, env: Env) { parsedUserAgent.getBrowser().name; const ifModifiedSince = request.headers.get("if-modified-since"); - const { newVisitor, newSession } = checkVisitorSession(ifModifiedSince); + const { newVisitor } = checkVisitorSession(ifModifiedSince); const modifiedDate = getNextModifiedDate( ifModifiedSince ? new Date(ifModifiedSince) : null, ); @@ -115,7 +103,7 @@ export function collectRequestHandler(request: Request, env: Env) { path: params.p, referrer: params.r, newVisitor: newVisitor ? 1 : 0, - newSession: newSession ? 1 : 0, + newSession: 0, // dead column bounce: newVisitor ? 1 : getBounce(modifiedDate), // user agent stuff userAgent: userAgent, diff --git a/app/analytics/query.ts b/app/analytics/query.ts index 71fbfe77..a39c1799 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -19,13 +19,12 @@ interface AnalyticsQueryResult< interface AnalyticsCountResult { views: number; - visits: number; visitors: number; bounces: number; } /** Given an AnalyticsCountResult object, and an object representing a row returned from - * CF Analytics Engine w/ counts grouped by isVisitor and isVisit, accumulate view, + * CF Analytics Engine w/ counts grouped by isVisitor, accumulate view, * visit, and visitor counts. */ function accumulateCountsFromRowResult( @@ -33,13 +32,9 @@ function accumulateCountsFromRowResult( row: { count: number; isVisitor: number; - isVisit: number; isBounce: number; }, ) { - if (row.isVisit == 1) { - counts.visits += Number(row.count); - } if (row.isVisitor == 1) { counts.visitors += Number(row.count); } @@ -214,9 +209,8 @@ export class AnalyticsEngineAPI { const filterStr = filtersToSql(filters); - // NOTE: when using toStartOfInterval, cannot group by other columns - // like double1 (isVisitor) or double2 (isSession/isVisit). This - // is just a limitation of Cloudflare Analytics Engine. + // NOTE: when using toStartOfInterval, cannot group by other columns like double1 (isVisitor). + // This is just a limitation of Cloudflare Analytics Engine. // -- but you can filter on them (using WHERE) // NOTE 2: Since CF AE doesn't support COALESCE, this query will not return @@ -309,19 +303,17 @@ export class AnalyticsEngineAPI { const query = ` SELECT SUM(_sample_interval) as count, ${ColumnMappings.newVisitor} as isVisitor, - ${ColumnMappings.newSession} as isVisit, ${ColumnMappings.bounce} as isBounce FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} ${filterStr} AND ${siteIdColumn} = '${siteId}' - GROUP BY isVisitor, isVisit, isBounce - ORDER BY isVisitor, isVisit, isBounce ASC`; + GROUP BY isVisitor, isBounce + ORDER BY isVisitor, isBounce ASC`; type SelectionSet = { count: number; isVisitor: number; - isVisit: number; isBounce: number; }; @@ -342,7 +334,6 @@ export class AnalyticsEngineAPI { const counts: AnalyticsCountResult = { views: 0, visitors: 0, - visits: 0, bounces: 0, }; @@ -445,21 +436,19 @@ export class AnalyticsEngineAPI { const query = ` SELECT ${_column}, ${ColumnMappings.newVisitor} as isVisitor, - ${ColumnMappings.newSession} as isVisit, ${ColumnMappings.bounce} as isBounce, SUM(_sample_interval) as count FROM metricsDataset WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql} AND ${ColumnMappings.siteId} = '${siteId}' ${filterStr} - GROUP BY ${_column}, ${ColumnMappings.newVisitor}, ${ColumnMappings.newSession}, ${ColumnMappings.bounce} + GROUP BY ${_column}, ${ColumnMappings.newVisitor}, ${ColumnMappings.bounce} ORDER BY count DESC LIMIT ${limit * page}`; type SelectionSet = { readonly count: number; readonly isVisitor: number; - readonly isVisit: number; readonly isBounce: number; } & Record< (typeof ColumnMappings)[T], @@ -493,7 +482,6 @@ export class AnalyticsEngineAPI { acc[key] = { views: 0, visitors: 0, - visits: 0, bounces: 0, } as AnalyticsCountResult; } diff --git a/app/routes/__tests__/dashboard.test.tsx b/app/routes/__tests__/dashboard.test.tsx index 4ba95e89..238ffd1b 100644 --- a/app/routes/__tests__/dashboard.test.tsx +++ b/app/routes/__tests__/dashboard.test.tsx @@ -217,7 +217,6 @@ describe("Dashboard route", () => { loader: () => { return json({ views: 0, - visits: 0, visitors: 0, }); }, @@ -270,7 +269,6 @@ describe("Dashboard route", () => { siteId: "example", sites: ["example"], views: 2133, - visits: 80, visitors: 33, viewsGroupedByInterval: [ ["2024-01-11 05:00:00", 0], @@ -304,7 +302,6 @@ describe("Dashboard route", () => { loader: () => { return json({ views: 2133, - visits: 80, visitors: 33, }); }, diff --git a/app/routes/__tests__/resources.stats.test.tsx b/app/routes/__tests__/resources.stats.test.tsx index 767ec63e..2b0371ae 100644 --- a/app/routes/__tests__/resources.stats.test.tsx +++ b/app/routes/__tests__/resources.stats.test.tsx @@ -5,7 +5,6 @@ describe("resources.stats loader", () => { test("returns formatted stats from analytics engine", async () => { const mockGetCounts = vi.fn().mockResolvedValue({ views: 1000, - visits: 500, visitors: 250, }); @@ -31,7 +30,6 @@ describe("resources.stats loader", () => { expect(data).toEqual({ views: 1000, - visits: 500, visitors: 250, }); }); diff --git a/app/routes/resources.stats.tsx b/app/routes/resources.stats.tsx index 5ade6bcc..0f09f0a8 100644 --- a/app/routes/resources.stats.tsx +++ b/app/routes/resources.stats.tsx @@ -17,7 +17,6 @@ export async function loader({ context, request }: LoaderFunctionArgs) { return json({ views: counts.views, - visits: counts.visits, visitors: counts.visitors, bounces: counts.bounces, }); @@ -36,7 +35,7 @@ export const StatsCard = ({ }) => { const dataFetcher = useFetcher(); - const { views, visits, visitors, bounces } = dataFetcher.data || {}; + const { views, visitors, bounces } = dataFetcher.data || {}; const countFormatter = Intl.NumberFormat("en", { notation: "compact" }); useEffect(() => { @@ -65,12 +64,6 @@ export const StatsCard = ({ {views ? countFormatter.format(views) : "-"} -
-
Visits
-
- {visits ? countFormatter.format(visits) : "-"} -
-
Visitors
From 98a67161eae5c4cf5440113e40e36aabd9ddfae0 Mon Sep 17 00:00:00 2001 From: Ben Vinegar <2153+benvinegar@users.noreply.github.com> Date: Wed, 11 Dec 2024 15:38:15 -0500 Subject: [PATCH 13/16] Improve variable naming --- app/analytics/collect.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/analytics/collect.ts b/app/analytics/collect.ts index efc745c1..82df4cab 100644 --- a/app/analytics/collect.ts +++ b/app/analytics/collect.ts @@ -11,7 +11,7 @@ function getMidnightDate(): Date { return midnight; } -function getNextModifiedDate(current: Date | null): Date { +function getNextLastModifiedDate(current: Date | null): Date { // in case date is an 'Invalid Date' if (current && isNaN(current.getTime())) { current = null; @@ -28,13 +28,16 @@ function getNextModifiedDate(current: Date | null): Date { return next; } -function getBounce(current: Date | null): number { - if (!current) { +function getBounceValue(nextLastModifiedDate: Date | null): number { + if (!nextLastModifiedDate) { return 0; } const midnight = getMidnightDate(); - const visits = (current.getTime() - midnight.getTime()) / 1000 - 1; + + // NOTE: minus one because this is the response last modified date + const visits = + (nextLastModifiedDate.getTime() - midnight.getTime()) / 1000 - 1; switch (visits) { case 0: @@ -93,7 +96,7 @@ export function collectRequestHandler(request: Request, env: Env) { const ifModifiedSince = request.headers.get("if-modified-since"); const { newVisitor } = checkVisitorSession(ifModifiedSince); - const modifiedDate = getNextModifiedDate( + const nextLastModifiedDate = getNextLastModifiedDate( ifModifiedSince ? new Date(ifModifiedSince) : null, ); @@ -104,7 +107,7 @@ export function collectRequestHandler(request: Request, env: Env) { referrer: params.r, newVisitor: newVisitor ? 1 : 0, newSession: 0, // dead column - bounce: newVisitor ? 1 : getBounce(modifiedDate), + bounce: newVisitor ? 1 : getBounceValue(nextLastModifiedDate), // user agent stuff userAgent: userAgent, browserName: parsedUserAgent.getBrowser().name, @@ -136,7 +139,7 @@ export function collectRequestHandler(request: Request, env: Env) { Expires: "Mon, 01 Jan 1990 00:00:00 GMT", "Cache-Control": "no-cache", Pragma: "no-cache", - "Last-Modified": modifiedDate.toUTCString(), + "Last-Modified": nextLastModifiedDate.toUTCString(), Tk: "N", // not tracking }, status: 200, From 0fb24047842007dafd57726780c78359b89782ea Mon Sep 17 00:00:00 2001 From: Ben Vinegar <2153+benvinegar@users.noreply.github.com> Date: Wed, 11 Dec 2024 18:27:14 -0500 Subject: [PATCH 14/16] Backwards compatibility for when bounce rate is unavailable --- app/analytics/__tests__/query.test.ts | 24 ++++++ app/analytics/query.ts | 52 +++++++++++ app/routes/__tests__/resources.stats.test.tsx | 86 ++++++++++++++++++- app/routes/resources.stats.tsx | 53 ++++++++++-- 4 files changed, 205 insertions(+), 10 deletions(-) diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index f078cfd5..18d8be89 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -339,6 +339,30 @@ describe("AnalyticsEngineAPI", () => { }); }); }); + + describe("getEarliestEvents", () => { + test("returns both earliest event and bounce dates when found", async () => { + const mockEventTimestamp = "2024-01-01T10:00:00Z"; + const mockBounceTimestamp = "2024-01-01T12:00:00Z"; + + // Mock responses for both queries + fetch.mockResolvedValueOnce( + createFetchResponse({ + ok: true, + data: [ + { earliestEvent: mockBounceTimestamp, isBounce: 1 }, + { earliestEvent: mockEventTimestamp, isBounce: 0 }, + ], + }), + ); + + const result = await api.getEarliestEvents("test-site"); + expect(result).toEqual({ + earliestEvent: new Date(mockEventTimestamp), + earliestBounce: new Date(mockBounceTimestamp), + }); + }); + }); }); describe("intervalToSql", () => { diff --git a/app/analytics/query.ts b/app/analytics/query.ts index a39c1799..74a4bb95 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -650,4 +650,56 @@ export class AnalyticsEngineAPI { ); return returnPromise; } + + async getEarliestEvents(siteId: string): Promise<{ + earliestEvent: Date | null; + earliestBounce: Date | null; + }> { + const query = ` + SELECT + MIN(timestamp) as earliestEvent, + ${ColumnMappings.bounce} as isBounce + FROM metricsDataset + WHERE ${ColumnMappings.siteId} = '${siteId}' + GROUP by isBounce + `; + + type SelectionSet = { + earliestEvent: string; + isBounce: number; + }; + const queryResult = this.query(query); + const returnPromise = new Promise<{ + earliestEvent: Date | null; + earliestBounce: Date | null; + }>((resolve, reject) => { + (async () => { + const response = await queryResult; + + if (!response.ok) { + reject(response.statusText); + return; + } + + const responseData = + (await response.json()) as AnalyticsQueryResult; + + const data = responseData.data; + + const earliestEvent = data.filter( + (row) => row["isBounce"] === 0, + )[0]["earliestEvent"]; + const earliestBounce = data.filter( + (row) => row["isBounce"] === 1, + )[0]["earliestEvent"]; + + resolve({ + earliestEvent: new Date(earliestEvent), + earliestBounce: new Date(earliestBounce), + }); + })(); + }); + + return returnPromise; + } } diff --git a/app/routes/__tests__/resources.stats.test.tsx b/app/routes/__tests__/resources.stats.test.tsx index 2b0371ae..192d0240 100644 --- a/app/routes/__tests__/resources.stats.test.tsx +++ b/app/routes/__tests__/resources.stats.test.tsx @@ -1,16 +1,33 @@ -import { describe, test, expect, vi } from "vitest"; +import { describe, test, expect, beforeEach, afterEach, vi } from "vitest"; import { loader } from "../resources.stats"; describe("resources.stats loader", () => { - test("returns formatted stats from analytics engine", async () => { - const mockGetCounts = vi.fn().mockResolvedValue({ + let mockGetCounts: any; + beforeEach(() => { + mockGetCounts = vi.fn().mockResolvedValue({ views: 1000, visitors: 250, + bounces: 125, + }); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + test("returns formatted stats from analytics engine", async () => { + vi.setSystemTime(new Date("2023-01-01T06:00:00").getTime()); + + const mockGetEarliestEvents = vi.fn().mockResolvedValue({ + // earliest event and earliest bounce are the same + earliestEvent: new Date("2023-01-01T00:00:00Z"), + earliestBounce: new Date("2023-01-01T00:00:00Z"), }); const context = { analyticsEngine: { getCounts: mockGetCounts, + getEarliestEvents: mockGetEarliestEvents, }, }; @@ -31,6 +48,69 @@ describe("resources.stats loader", () => { expect(data).toEqual({ views: 1000, visitors: 250, + bounceRate: "50%", + }); + }); + + test("if bounce data isn't complete for the given interval, show n/a", async () => { + // set system time as jan 8th + vi.setSystemTime(new Date("2023-01-08T00:00:00").getTime()); + + const mockGetEarliestEvents = vi.fn().mockResolvedValue({ + earliestEvent: new Date("2023-01-01T00:00:00Z"), + earliestBounce: new Date("2023-01-04T00:00:00Z"), // Jan 4 + }); + + const context = { + analyticsEngine: { + getCounts: mockGetCounts, + getEarliestEvents: mockGetEarliestEvents, + }, + }; + + const request = new Request( + // 7 day interval (specified in query string) + "https://example.com/resources/stats?site=test-site&interval=7d&timezone=UTC", + ); + + const response = await loader({ context, request } as any); + const data = await response.json(); + + expect(data).toEqual({ + views: 1000, + visitors: 250, + bounceRate: "n/a", + }); + }); + + test("if bounce data *IS* complete for the given interval, show it", async () => { + // set system time as jan 8th + vi.setSystemTime(new Date("2023-01-08T00:00:00").getTime()); + + const mockGetEarliestEvents = vi.fn().mockResolvedValue({ + earliestEvent: new Date("2023-01-01T00:00:00Z"), + earliestBounce: new Date("2023-01-04T00:00:00Z"), // Jan 4 -- well before Jan 8th minus 1 day interval + }); + + const context = { + analyticsEngine: { + getCounts: mockGetCounts, + getEarliestEvents: mockGetEarliestEvents, + }, + }; + + const request = new Request( + // 1 day interval (specified in query string) + "https://example.com/resources/stats?site=test-site&interval=1d&timezone=UTC", + ); + + const response = await loader({ context, request } as any); + const data = await response.json(); + + expect(data).toEqual({ + views: 1000, + visitors: 250, + bounceRate: "50%", }); }); }); diff --git a/app/routes/resources.stats.tsx b/app/routes/resources.stats.tsx index 0f09f0a8..eacdce9c 100644 --- a/app/routes/resources.stats.tsx +++ b/app/routes/resources.stats.tsx @@ -1,6 +1,10 @@ import type { LoaderFunctionArgs } from "@remix-run/cloudflare"; import { json } from "@remix-run/cloudflare"; -import { getFiltersFromSearchParams, paramsFromUrl } from "~/lib/utils"; +import { + getDateTimeRange, + getFiltersFromSearchParams, + paramsFromUrl, +} from "~/lib/utils"; import { useEffect } from "react"; import { useFetcher } from "@remix-run/react"; import { Card } from "~/components/ui/card"; @@ -13,12 +17,49 @@ export async function loader({ context, request }: LoaderFunctionArgs) { const tz = url.searchParams.get("timezone") || "UTC"; const filters = getFiltersFromSearchParams(url.searchParams); + // intentionally parallelize queries by deferring await + const earliestEvents = analyticsEngine.getEarliestEvents(site); const counts = await analyticsEngine.getCounts(site, interval, tz, filters); + const { earliestEvent, earliestBounce } = await earliestEvents; + const { startDate } = getDateTimeRange(interval, tz); + + // FOR BACKWARDS COMPATIBILITY, ONLY CALCULATE BOUNCE RATE IF WE HAVE + // DATE FOR THE ENTIRE QUERY PERIOD + // ----------------------------------------------------------------------------- + // bounce rate is a later-introduced metric that may not have been recorded for + // the full duration of the queried Counterscale dataset (not possible to backfill + // data we dont have!) + + // so, cannot reliably show "bounce rate" if bounce data was unavailable for a portion + // of the query period + + // to figure if we can give an answer or not, we inspect the earliest bounce/earliest event data, + // and decide if our dataset is "complete" for the given interval + let bounceRate; + + if ( + counts.visitors > 0 && + earliestBounce !== null && + earliestEvent !== null && + (earliestEvent.getTime() == earliestBounce.getTime() || // earliest event recorded a bounce -- any query is fine + earliestBounce < startDate) // earliest bounce occurred before start of query period -- this query is fine + ) { + bounceRate = (counts.bounces / counts.visitors).toLocaleString( + "en-US", + { + style: "percent", + minimumFractionDigits: 0, + }, + ); + } else { + bounceRate = "n/a"; + } + return json({ views: counts.views, visitors: counts.visitors, - bounces: counts.bounces, + bounceRate: bounceRate, }); } @@ -35,7 +76,7 @@ export const StatsCard = ({ }) => { const dataFetcher = useFetcher(); - const { views, visitors, bounces } = dataFetcher.data || {}; + const { views, visitors, bounceRate } = dataFetcher.data || {}; const countFormatter = Intl.NumberFormat("en", { notation: "compact" }); useEffect(() => { @@ -71,10 +112,8 @@ export const StatsCard = ({
-
Bounces
-
- {bounces ? countFormatter.format(bounces) : "-"} -
+
Bounce Rate
+
{bounceRate}
From 168c06bb96748614891b0b79657ec1c35f929824 Mon Sep 17 00:00:00 2001 From: Ben Vinegar <2153+benvinegar@users.noreply.github.com> Date: Wed, 11 Dec 2024 21:57:05 -0500 Subject: [PATCH 15/16] Handle getEarliestEvent when no bounce events --- app/analytics/__tests__/query.test.ts | 18 ++++++++++++++++++ app/analytics/query.ts | 17 +++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/app/analytics/__tests__/query.test.ts b/app/analytics/__tests__/query.test.ts index 18d8be89..ade4cc55 100644 --- a/app/analytics/__tests__/query.test.ts +++ b/app/analytics/__tests__/query.test.ts @@ -362,6 +362,24 @@ describe("AnalyticsEngineAPI", () => { earliestBounce: new Date(mockBounceTimestamp), }); }); + + test("returns only earliest event when no bounces found", async () => { + const mockEventTimestamp = "2024-01-01T10:00:00Z"; + + // Mock responses for both queries + fetch.mockResolvedValueOnce( + createFetchResponse({ + ok: true, + data: [{ earliestEvent: mockEventTimestamp, isBounce: 0 }], + }), + ); + + const result = await api.getEarliestEvents("test-site"); + expect(result).toEqual({ + earliestEvent: new Date(mockEventTimestamp), + earliestBounce: null, + }); + }); }); }); diff --git a/app/analytics/query.ts b/app/analytics/query.ts index 74a4bb95..51ecd4c2 100644 --- a/app/analytics/query.ts +++ b/app/analytics/query.ts @@ -686,16 +686,21 @@ export class AnalyticsEngineAPI { const data = responseData.data; - const earliestEvent = data.filter( + const earliestEvent = data.find( (row) => row["isBounce"] === 0, - )[0]["earliestEvent"]; - const earliestBounce = data.filter( + )?.earliestEvent; + + const earliestBounce = data.find( (row) => row["isBounce"] === 1, - )[0]["earliestEvent"]; + )?.earliestEvent; resolve({ - earliestEvent: new Date(earliestEvent), - earliestBounce: new Date(earliestBounce), + earliestEvent: earliestEvent + ? new Date(earliestEvent) + : null, + earliestBounce: earliestBounce + ? new Date(earliestBounce) + : null, }); })(); }); From bd4c6879c8533f0106a16e71bb00347d326cda82 Mon Sep 17 00:00:00 2001 From: Ben Vinegar <2153+benvinegar@users.noreply.github.com> Date: Wed, 11 Dec 2024 22:09:00 -0500 Subject: [PATCH 16/16] Clearer code re: missing bounce data --- app/routes/__tests__/resources.stats.test.tsx | 11 +++-- app/routes/resources.stats.tsx | 47 +++++++++---------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/app/routes/__tests__/resources.stats.test.tsx b/app/routes/__tests__/resources.stats.test.tsx index 192d0240..f0629e3e 100644 --- a/app/routes/__tests__/resources.stats.test.tsx +++ b/app/routes/__tests__/resources.stats.test.tsx @@ -48,11 +48,12 @@ describe("resources.stats loader", () => { expect(data).toEqual({ views: 1000, visitors: 250, - bounceRate: "50%", + bounceRate: 0.5, + hasSufficientBounceData: true, }); }); - test("if bounce data isn't complete for the given interval, show n/a", async () => { + test("if bounce data isn't complete for the given interval, hasSufficientBounceData is false", async () => { // set system time as jan 8th vi.setSystemTime(new Date("2023-01-08T00:00:00").getTime()); @@ -79,7 +80,8 @@ describe("resources.stats loader", () => { expect(data).toEqual({ views: 1000, visitors: 250, - bounceRate: "n/a", + bounceRate: 0.5, + hasSufficientBounceData: false, }); }); @@ -110,7 +112,8 @@ describe("resources.stats loader", () => { expect(data).toEqual({ views: 1000, visitors: 250, - bounceRate: "50%", + bounceRate: 0.5, + hasSufficientBounceData: true, }); }); }); diff --git a/app/routes/resources.stats.tsx b/app/routes/resources.stats.tsx index eacdce9c..2891260d 100644 --- a/app/routes/resources.stats.tsx +++ b/app/routes/resources.stats.tsx @@ -24,42 +24,32 @@ export async function loader({ context, request }: LoaderFunctionArgs) { const { earliestEvent, earliestBounce } = await earliestEvents; const { startDate } = getDateTimeRange(interval, tz); - // FOR BACKWARDS COMPATIBILITY, ONLY CALCULATE BOUNCE RATE IF WE HAVE - // DATE FOR THE ENTIRE QUERY PERIOD + // FOR BACKWARDS COMPAT, ONLY SHOW BOUNCE RATE IF WE HAVE DATE FOR THE ENTIRE QUERY PERIOD // ----------------------------------------------------------------------------- - // bounce rate is a later-introduced metric that may not have been recorded for + // Bounce rate is a later-introduced metric that may not have been recorded for // the full duration of the queried Counterscale dataset (not possible to backfill // data we dont have!) - // so, cannot reliably show "bounce rate" if bounce data was unavailable for a portion - // of the query period + // So, cannot reliably show "bounce rate" if bounce data was unavailable for a portion + // of the query period. - // to figure if we can give an answer or not, we inspect the earliest bounce/earliest event data, - // and decide if our dataset is "complete" for the given interval - let bounceRate; + // To figure out if we can give an answer or not, we inspect the earliest bounce/earliest event + // data recorded, and determine if our dataset is "complete" for the given query interval. - if ( - counts.visitors > 0 && + const hasSufficientBounceData = earliestBounce !== null && earliestEvent !== null && (earliestEvent.getTime() == earliestBounce.getTime() || // earliest event recorded a bounce -- any query is fine - earliestBounce < startDate) // earliest bounce occurred before start of query period -- this query is fine - ) { - bounceRate = (counts.bounces / counts.visitors).toLocaleString( - "en-US", - { - style: "percent", - minimumFractionDigits: 0, - }, - ); - } else { - bounceRate = "n/a"; - } + earliestBounce < startDate); // earliest bounce occurred before start of query period -- this query is fine + + const bounceRate = + counts.visitors > 0 ? counts.bounces / counts.visitors : undefined; return json({ views: counts.views, visitors: counts.visitors, bounceRate: bounceRate, + hasSufficientBounceData, }); } @@ -76,7 +66,8 @@ export const StatsCard = ({ }) => { const dataFetcher = useFetcher(); - const { views, visitors, bounceRate } = dataFetcher.data || {}; + const { views, visitors, bounceRate, hasSufficientBounceData } = + dataFetcher.data || {}; const countFormatter = Intl.NumberFormat("en", { notation: "compact" }); useEffect(() => { @@ -113,7 +104,15 @@ export const StatsCard = ({
Bounce Rate
-
{bounceRate}
+ {hasSufficientBounceData ? ( +
+ {bounceRate !== undefined + ? `${Math.round(bounceRate * 100)}%` + : "-"} +
+ ) : ( +
n/a
+ )}