Skip to content

Commit

Permalink
Enhance Date Interval Handling and Add "Yesterday" Interval Support (#98
Browse files Browse the repository at this point in the history 
)

* Add yesterday functionality to the dashboard

* Remove added utc

* Add removed utc

* endDateTime optional parameter

* Move end date calc up one level

* Clean up dayjs use

* Clean up sql query
  • Loading branch information
@DylanPetrey
DylanPetrey authored Oct 26, 2024
1 parent 0f9e960 commit 56518f8
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 27 deletions.
45 changes: 36 additions & 9 deletions app/analytics/__tests__/query.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ describe("AnalyticsEngineAPI", () => {
"example.com",
"DAY",
new Date("2024-01-11 00:00:00"), // local time (because tz also passed)
new Date(),
"America/New_York",
);

Expand All @@ -106,6 +107,7 @@ describe("AnalyticsEngineAPI", () => {
"example.com",
"DAY",
new Date("2024-01-13 00:00:00"), // local time (because tz also passed)
new Date(),
"America/New_York",
);
expect(result2).toEqual([
Expand Down Expand Up @@ -148,6 +150,7 @@ describe("AnalyticsEngineAPI", () => {
"example.com",
"HOUR",
new Date("2024-01-17 05:00:00"), // local time (because tz also passed)
new Date(),
"America/New_York",
);

Expand Down Expand Up @@ -323,7 +326,7 @@ describe("AnalyticsEngineAPI", () => {
"double1 as isVisitor, " +
"double2 as isVisit, " +
"SUM(_sample_interval) as count " +
"FROM metricsDataset WHERE timestamp > NOW() - INTERVAL '7' DAY AND blob8 = 'example.com' AND blob4 = 'CA' " +
"FROM metricsDataset WHERE timestamp >= NOW() - INTERVAL '7' DAY AND timestamp < NOW() AND blob8 = 'example.com' AND blob4 = 'CA' " +
"GROUP BY blob4, double1, double2 " +
"ORDER BY count DESC LIMIT 10",
);
Expand All @@ -345,17 +348,41 @@ describe("intervalToSql", () => {

// test intervalToSql
test("should return the proper sql interval for 1d, 30d, 90d, etc (days)", () => {
expect(intervalToSql("1d")).toBe("NOW() - INTERVAL '1' DAY");
expect(intervalToSql("30d")).toBe("NOW() - INTERVAL '30' DAY");
expect(intervalToSql("90d")).toBe("NOW() - INTERVAL '90' DAY");
expect(intervalToSql("1d")).toStrictEqual({
startIntervalSql: "NOW() - INTERVAL '1' DAY",
endIntervalSql: "NOW()",
});
expect(intervalToSql("30d")).toStrictEqual({
startIntervalSql: "NOW() - INTERVAL '30' DAY",
endIntervalSql: "NOW()",
});
expect(intervalToSql("90d")).toStrictEqual({
startIntervalSql: "NOW() - INTERVAL '90' DAY",
endIntervalSql: "NOW()",
});
});

test("should return the proper tz-adjusted sql interval for 'today'", () => {
expect(intervalToSql("today", "America/New_York")).toBe(
"toDateTime('2024-04-29 04:00:00')",
);
expect(intervalToSql("today", "America/Los_Angeles")).toBe(
"toDateTime('2024-04-29 07:00:00')",
expect(intervalToSql("today", "America/New_York")).toStrictEqual({
startIntervalSql: "toDateTime('2024-04-29 04:00:00')",
endIntervalSql: "NOW()",
});
expect(intervalToSql("today", "America/Los_Angeles")).toStrictEqual({
startIntervalSql: "toDateTime('2024-04-29 07:00:00')",
endIntervalSql: "NOW()",
});
});

test("should return the proper tz-adjusted sql interval for 'yesterday'", () => {
expect(intervalToSql("yesterday", "America/New_York")).toStrictEqual({
startIntervalSql: "toDateTime('2024-04-28 04:00:00')",
endIntervalSql: "toDateTime('2024-04-29 04:00:00')",
});
expect(intervalToSql("yesterday", "America/Los_Angeles")).toStrictEqual(
{
startIntervalSql: "toDateTime('2024-04-28 07:00:00')",
endIntervalSql: "toDateTime('2024-04-29 07:00:00')",
},
);
});
});
56 changes: 40 additions & 16 deletions app/analytics/query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,22 +45,30 @@ function accumulateCountsFromRowResult(
}

export function intervalToSql(interval: string, tz?: string) {
let intervalSql = "";
let startIntervalSql = "";
let endIntervalSql = "";
switch (interval) {
case "today":
// example: toDateTime('2024-01-07 00:00:00', 'America/New_York')
intervalSql = `toDateTime('${dayjs().tz(tz).startOf("day").utc().format("YYYY-MM-DD HH:mm:ss")}')`;
startIntervalSql = `toDateTime('${dayjs().tz(tz).startOf("day").utc().format("YYYY-MM-DD HH:mm:ss")}')`;
endIntervalSql = "NOW()";
break;
case "yesterday":
startIntervalSql = `toDateTime('${dayjs().tz(tz).startOf("day").utc().subtract(1, "day").format("YYYY-MM-DD HH:mm:ss")}')`;
endIntervalSql = `toDateTime('${dayjs().tz(tz).startOf("day").utc().format("YYYY-MM-DD HH:mm:ss")}')`;
break;
case "1d":
case "7d":
case "30d":
case "90d":
intervalSql = `NOW() - INTERVAL '${interval.split("d")[0]}' DAY`;
startIntervalSql = `NOW() - INTERVAL '${interval.split("d")[0]}' DAY`;
endIntervalSql = "NOW()";
break;
default:
intervalSql = `NOW() - INTERVAL '1' DAY`;
startIntervalSql = `NOW() - INTERVAL '1' DAY`;
endIntervalSql = "NOW()";
}
return intervalSql;
return { startIntervalSql, endIntervalSql };
}

/**
Expand All @@ -77,6 +85,7 @@ export function intervalToSql(interval: string, tz?: string) {
function generateEmptyRowsOverInterval(
intervalType: "DAY" | "HOUR",
startDateTime: Date,
endDateTime: Date,
tz?: string,
): { [key: string]: number } {
if (!tz) {
Expand All @@ -97,7 +106,7 @@ function generateEmptyRowsOverInterval(
// out how to get vitest/mock dates to recreate DST changes.
// See: https://github.com/benvinegar/counterscale/pull/62

while (startDateTime.getTime() < Date.now()) {
while (startDateTime.getTime() < endDateTime.getTime()) {
const key = dayjs(startDateTime).utc().format("YYYY-MM-DD HH:mm:ss");
initialRows[key] = 0;

Expand Down Expand Up @@ -173,6 +182,7 @@ export class AnalyticsEngineAPI {
siteId: string,
intervalType: "DAY" | "HOUR",
startDateTime: Date, // start date/time in local timezone
endDateTime: Date, // end date/time in local timezone
tz?: string, // local timezone
filters: SearchFilters = {},
) {
Expand All @@ -190,6 +200,7 @@ export class AnalyticsEngineAPI {
const initialRows = generateEmptyRowsOverInterval(
intervalType,
startDateTime,
endDateTime,
tz,
);

Expand All @@ -206,6 +217,7 @@ export class AnalyticsEngineAPI {
// and merge them with the results.

const localStartTime = dayjs(startDateTime).tz(tz).utc();
const localEndTime = dayjs(endDateTime).tz(tz).utc();

const query = `
SELECT SUM(_sample_interval) as count,
Expand All @@ -215,9 +227,9 @@ export class AnalyticsEngineAPI {
/* output as UTC */
toDateTime(_bucket, 'Etc/UTC') as bucket
FROM metricsDataset
WHERE timestamp > toDateTime('${localStartTime.format("YYYY-MM-DD HH:mm:ss")}')
WHERE timestamp >= toDateTime('${localStartTime.format("YYYY-MM-DD HH:mm:ss")}')
AND timestamp < toDateTime('${localEndTime.format("YYYY-MM-DD HH:mm:ss")}')
AND ${ColumnMappings.siteId} = '${siteId}'
${filterStr}
GROUP BY _bucket
Expand Down Expand Up @@ -279,7 +291,10 @@ export class AnalyticsEngineAPI {
// defaults to 1 day if not specified
const siteIdColumn = ColumnMappings["siteId"];

const intervalSql = intervalToSql(interval, tz);
const { startIntervalSql, endIntervalSql } = intervalToSql(
interval,
tz,
);

const filterStr = filtersToSql(filters);

Expand All @@ -288,7 +303,7 @@ export class AnalyticsEngineAPI {
${ColumnMappings.newVisitor} as isVisitor,
${ColumnMappings.newSession} as isVisit
FROM metricsDataset
WHERE timestamp > ${intervalSql}
WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql}
${filterStr}
AND ${siteIdColumn} = '${siteId}'
GROUP BY isVisitor, isVisit
Expand Down Expand Up @@ -341,15 +356,18 @@ export class AnalyticsEngineAPI {
page: number = 1,
limit: number = 10,
) {
const intervalSql = intervalToSql(interval, tz);
const { startIntervalSql, endIntervalSql } = intervalToSql(
interval,
tz,
);

const filterStr = filtersToSql(filters);

const _column = ColumnMappings[column];
const query = `
SELECT ${_column}, SUM(_sample_interval) as count
FROM metricsDataset
WHERE timestamp > ${intervalSql}
WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql}
AND ${ColumnMappings.newVisitor} = 1
AND ${ColumnMappings.siteId} = '${siteId}'
${filterStr}
Expand Down Expand Up @@ -405,7 +423,10 @@ export class AnalyticsEngineAPI {
page: number = 1,
limit: number = 10,
): Promise<Record<string, AnalyticsCountResult>> {
const intervalSql = intervalToSql(interval, tz);
const { startIntervalSql, endIntervalSql } = intervalToSql(
interval,
tz,
);

const filterStr = filtersToSql(filters);

Expand All @@ -416,7 +437,7 @@ export class AnalyticsEngineAPI {
${ColumnMappings.newSession} as isVisit,
SUM(_sample_interval) as count
FROM metricsDataset
WHERE timestamp > ${intervalSql}
WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql}
AND ${ColumnMappings.siteId} = '${siteId}'
${filterStr}
GROUP BY ${_column}, ${ColumnMappings.newVisitor}, ${ColumnMappings.newSession}
Expand Down Expand Up @@ -584,13 +605,16 @@ export class AnalyticsEngineAPI {

limit = limit || 10;

const intervalSql = intervalToSql(interval, tz);
const { startIntervalSql, endIntervalSql } = intervalToSql(
interval,
tz,
);

const query = `
SELECT SUM(_sample_interval) as count,
${ColumnMappings.siteId} as siteId
FROM metricsDataset
WHERE timestamp > ${intervalSql}
WHERE timestamp >= ${startIntervalSql} AND timestamp < ${endIntervalSql}
GROUP BY siteId
ORDER BY count DESC
LIMIT ${limit}
Expand Down
9 changes: 9 additions & 0 deletions app/routes/dashboard.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ export const loader = async ({ context, request }: LoaderFunctionArgs) => {
let intervalType: "DAY" | "HOUR" = "DAY";
switch (interval) {
case "today":
case "yesterday":
case "1d":
intervalType = "HOUR";
break;
Expand All @@ -107,8 +108,12 @@ export const loader = async ({ context, request }: LoaderFunctionArgs) => {
// get start date in the past by subtracting interval * type

let localDateTime = dayjs().utc();
let localEndDateTime: dayjs.Dayjs | undefined;
if (interval === "today") {
localDateTime = localDateTime.tz(tz).startOf("day");
} else if (interval === "yesterday") {
localDateTime = localDateTime.tz(tz).startOf("day").subtract(1, "day");
localEndDateTime = localDateTime.endOf("day").add(2, "ms");
} else {
const daysAgo = Number(interval.split("d")[0]);
if (intervalType === "DAY") {
Expand All @@ -123,10 +128,13 @@ export const loader = async ({ context, request }: LoaderFunctionArgs) => {
}
}

if (!localEndDateTime) localEndDateTime = dayjs().utc().tz(tz);

const viewsGroupedByInterval = analyticsEngine.getViewsGroupedByInterval(
actualSiteId,
intervalType,
localDateTime.toDate(),
localEndDateTime.toDate(),
tz,
filters,
);
Expand Down Expand Up @@ -249,6 +257,7 @@ export default function Dashboard() {
</SelectTrigger>
<SelectContent>
<SelectItem value="today">Today</SelectItem>
<SelectItem value="yesterday">Yesterday</SelectItem>
<SelectItem value="1d">24 hours</SelectItem>
<SelectItem value="7d">7 days</SelectItem>
<SelectItem value="30d">30 days</SelectItem>
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 56518f8

Please sign in to comment.