Skip to content

Commit

Permalink
Turn cronjob into long-running interruptible task
Browse files Browse the repository at this point in the history
This makes the breach-alerts cronjob explicitly interruptible via
SIGINT, and adds a script to turn it into a long-running task,
instead of a script meant for periodical execution. This should
enable scaling it based on the size of the PubSub queue.
  • Loading branch information
Vinnl committed Feb 24, 2025
1 parent 415e321 commit ec8741e
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 4 deletions.
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"dev:cron:monthly-activity-free": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/monthlyActivityFree.tsx",
"dev:cron:monthly-activity-plus": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/monthlyActivityPlus.tsx",
"dev:cron:breach-alerts": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/emailBreachAlerts.tsx",
"dev:task:breach-alerts": "LOOP='true' tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/emailBreachAlerts.tsx",
"dev:cron:db-delete-unverified-subscribers": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/deleteUnverifiedSubscribers.ts",
"dev:cron:db-pull-breaches": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/syncBreaches.ts",
"dev:cron:db-pull-data-brokers": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/syncOnerepDataBrokers.ts",
Expand All @@ -34,6 +35,7 @@
"cron:monthly-activity-free": "node dist/scripts/cronjobs/monthlyActivityFree.js",
"cron:monthly-activity-plus": "node dist/scripts/cronjobs/monthlyActivityPlus.js",
"cron:breach-alerts": "node dist/scripts/cronjobs/emailBreachAlerts.js",
"task:breach-alerts": "LOOP='true' node dist/scripts/cronjobs/emailBreachAlerts.js",
"cron:db-delete-unverified-subscribers": "node dist/scripts/cronjobs/deleteUnverifiedSubscribers.js",
"cron:db-pull-breaches": "node dist/scripts/cronjobs/syncBreaches.js",
"cron:db-pull-data-brokers": "node dist/scripts/cronjobs/syncOnerepDataBrokers.js",
Expand Down
24 changes: 20 additions & 4 deletions src/scripts/cronjobs/emailBreachAlerts.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import Sentry from "@sentry/nextjs";

import process from "node:process";
import * as pubsub from "@google-cloud/pubsub";
import * as grpc from "@grpc/grpc-js";
import type { SubscriberClient } from "@google-cloud/pubsub/build/src/v1/subscriber_client.js";
Expand Down Expand Up @@ -392,11 +393,26 @@ async function pullMessages() {

return [subClient, response.receivedMessages] as const;
}
async function init() {
await initEmail();

async function run() {
const [subClient, receivedMessages] = await pullMessages();
await poll(subClient, receivedMessages ?? []);

// We're transitioning this into a long-running process,
// but to ensure a smooth transition, allow keeping the cronjob behaviour:
if (process.env.LOOP === "true") {
logger.info("Processed batch, starting next one.");
await run();
}
}

async function init() {
process.on("SIGINT", () => {
throw new Error("SIGINT received, exiting...");
});
await initEmail();

await run();
}

if (process.env.NODE_ENV !== "test") {
Expand All @@ -408,7 +424,7 @@ if (process.env.NODE_ENV !== "test") {
);
}
})
.catch((err) => console.error(err))
.catch((err) => console.error("breach-alerts ended with an error:", err))
.finally(async () => {
// Tear down knex connection pools
await knexSubscribers.destroy();
Expand All @@ -420,7 +436,7 @@ if (process.env.NODE_ENV !== "test") {
monitorSlug: SENTRY_SLUG,
status: "ok",
});
setTimeout(process.exit, 1000);
setTimeout(() => process.exit(0), 1000);
});
}
/* c8 ignore stop */
Expand Down

0 comments on commit ec8741e

Please sign in to comment.