Skip to content

Commit

Permalink
Time out export jobs after 30 seconds.
Browse files Browse the repository at this point in the history
  • Loading branch information
NoahSaso committed Oct 18, 2023
1 parent e46f102 commit 2646b85
Showing 1 changed file with 41 additions and 24 deletions.
65 changes: 41 additions & 24 deletions src/scripts/export/workers/export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,30 +24,47 @@ export const makeExportWorker: ExportWorkerMaker<{

return {
queueName: QueueName.Export,
processor: async ({ data: { data } }) => {
// Group data by handler.
const groupedData = data.reduce(
(acc, { handler, data }) => ({
...acc,
[handler]: (acc[handler] || []).concat(data),
}),
{} as Record<string, any[]>
)

// Process data.
for (const { name, handler } of handlers) {
const events = groupedData[name]
if (!events?.length) {
continue
}
processor: ({ data: { data } }) =>
new Promise<void>(async (resolve, reject) => {
// Time out if takes more than 30 seconds.
let timeout: NodeJS.Timeout | null = setTimeout(() => {
timeout = null
reject(new Error('Export timed out after 30 seconds.'))
}, 30000)

try {
// Group data by handler.
const groupedData = data.reduce(
(acc, { handler, data }) => ({
...acc,
[handler]: (acc[handler] || []).concat(data),
}),
{} as Record<string, any[]>
)

// Process data.
for (const { name, handler } of handlers) {
const events = groupedData[name]
if (!events?.length) {
continue
}

// Retry 3 times with exponential backoff starting at 100ms delay.
await retry(handler.process, [events], {
retriesMax: 3,
exponential: true,
interval: 100,
})
}
},
// Retry 3 times with exponential backoff starting at 100ms delay.
await retry(handler.process, [events], {
retriesMax: 3,
exponential: true,
interval: 100,
})
}

resolve()
} catch (err) {
reject(err)
} finally {
if (timeout !== null) {
clearTimeout(timeout)
}
}
}),
}
}

0 comments on commit 2646b85

Please sign in to comment.