feat: Add maxDuration to tasks (#1377)

* WIP * Get max duration working on deployed runs * Actually set the timed out runs to status = TIMED_OUT * The client status for TIMED_OUT is now MAX_DURATION_EXCEEDED * New TimedOutIcon * Added new timedout icon * Add ability to opt-out of maxDuration with timeout.None * MAX_DURATION_EXCEEDED -> TIMED_OUT * changeset * Improved styling for the status tooltip content --------- Co-authored-by: James Ritchie <[email protected]>
triggerdotdev · Oct 3, 2024 · 6d08842 · 6d08842
1 parent 665ccf8
commit 6d08842
Show file tree

Hide file tree

Showing 55 changed files with 624 additions and 87 deletions.
diff --git a/.changeset/tiny-forks-remember.md b/.changeset/tiny-forks-remember.md
@@ -0,0 +1,7 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Adding maxDuration to tasks to allow timing out runs after they exceed a certain number of seconds
diff --git a/apps/webapp/app/assets/icons/TimedOutIcon.tsx b/apps/webapp/app/assets/icons/TimedOutIcon.tsx
@@ -0,0 +1,19 @@
+export function TimedOutIcon({ className }: { className?: string }) {
+  return (
+    <svg
+      className={className}
+      width="20"
+      height="20"
+      viewBox="0 0 20 20"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M9 2H8V3H9V4.07089C5.60771 4.55612 3 7.47353 3 11C3 14.866 6.13401 18 10 18C13.866 18 17 14.866 17 11C17 7.47353 14.3923 4.55612 11 4.07089V3H12V2H11H9ZM13.7603 3.36575C14.7218 2.40422 16.2807 2.40422 17.2422 3.36575C18.2038 4.32727 18.2038 5.8862 17.2422 6.84772L17.1462 6.94375C17.1251 6.96488 17.0908 6.96488 17.0697 6.94375L13.6642 3.53831C13.6431 3.51717 13.6431 3.4829 13.6642 3.46177L13.7603 3.36575ZM6.28876 3.58524C6.33584 3.53816 6.33584 3.46184 6.28876 3.41476L6.23971 3.36571C5.27819 2.40419 3.71925 2.40419 2.75773 3.36571C1.79621 4.32723 1.79621 5.88616 2.75773 6.84769L2.80678 6.89674C2.85386 6.94382 2.93019 6.94381 2.97726 6.89674L6.28876 3.58524ZM14.5858 17L16 15.5858L16.7071 16.2929C17.0976 16.6834 17.0976 17.3166 16.7071 17.7071C16.3166 18.0976 15.6834 18.0976 15.2929 17.7071L14.5858 17ZM5.42297 17L4.00875 15.5858L3.30165 16.2929C2.91112 16.6834 2.91112 17.3166 3.30165 17.7071C3.69217 18.0977 4.32534 18.0977 4.71586 17.7071L5.42297 17ZM6.29289 7.29289C6.68342 6.90237 7.31658 6.90237 7.70711 7.29289L10 9.58579L12.2929 7.29289C12.6834 6.90237 13.3166 6.90237 13.7071 7.29289C14.0976 7.68342 14.0976 8.31658 13.7071 8.70711L11.4142 11L13.7071 13.2929C14.0976 13.6834 14.0976 14.3166 13.7071 14.7071C13.3166 15.0976 12.6834 15.0976 12.2929 14.7071L10 12.4142L7.70711 14.7071C7.31658 15.0976 6.68342 15.0976 6.29289 14.7071C5.90237 14.3166 5.90237 13.6834 6.29289 13.2929L8.58579 11L6.29289 8.70711C5.90237 8.31658 5.90237 7.68342 6.29289 7.29289Z"
+        fill="currentColor"
+      />
+    </svg>
+  );
+}
diff --git a/apps/webapp/app/components/primitives/Select.tsx b/apps/webapp/app/components/primitives/Select.tsx
@@ -613,7 +613,7 @@ export function SelectPopover({
         "z-50 flex flex-col overflow-clip rounded border border-charcoal-700 bg-background-bright shadow-md outline-none animate-in fade-in-40",
         "min-w-[max(180px,calc(var(--popover-anchor-width)+0.5rem))]",
         "max-w-[min(480px,var(--popover-available-width))]",
-        "max-h-[min(480px,var(--popover-available-height))]",
+        "max-h-[min(520px,var(--popover-available-height))]",
         "origin-[var(--popover-transform-origin)]",
         className
       )}

diff --git a/apps/webapp/app/components/runs/v3/RunInspector.tsx b/apps/webapp/app/components/runs/v3/RunInspector.tsx
@@ -324,6 +324,12 @@ export function RunInspector({
                     </Property.Value>
                   </Property.Item>
                 )}
+                <Property.Item>
+                  <Property.Label>Max duration</Property.Label>
+                  <Property.Value>
+                    {run.maxDurationInSeconds ? `${run.maxDurationInSeconds}s` : "–"}
+                  </Property.Value>
+                </Property.Item>
                 <Property.Item>
                   <Property.Label>Run invocation cost</Property.Label>
                   <Property.Value>

diff --git a/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx b/apps/webapp/app/components/runs/v3/TaskRunStatus.tsx
@@ -8,12 +8,14 @@ import {
   NoSymbolIcon,
   PauseCircleIcon,
   RectangleStackIcon,
+  StopIcon,
   TrashIcon,
   XCircleIcon,
 } from "@heroicons/react/20/solid";
 import { TaskRunStatus } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { SnowflakeIcon } from "lucide-react";
+import { TimedOutIcon } from "~/assets/icons/TimedOutIcon";
 import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";
 
@@ -27,6 +29,7 @@ export const allTaskRunStatuses = [
   "COMPLETED_SUCCESSFULLY",
   "CANCELED",
   "COMPLETED_WITH_ERRORS",
+  "TIMED_OUT",
   "CRASHED",
   "PAUSED",
   "INTERRUPTED",
@@ -44,6 +47,7 @@ export const filterableTaskRunStatuses = [
   "COMPLETED_SUCCESSFULLY",
   "CANCELED",
   "COMPLETED_WITH_ERRORS",
+  "TIMED_OUT",
   "CRASHED",
   "INTERRUPTED",
   "SYSTEM_FAILURE",
@@ -65,6 +69,7 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   PAUSED: "Task has been paused by the user",
   CRASHED: "Task has crashed and won't be retried",
   EXPIRED: "Task has surpassed its ttl and won't be executed",
+  TIMED_OUT: "Task has reached its maxDuration and has been stopped",
 };
 
 export const QUEUED_STATUSES = [
@@ -140,6 +145,8 @@ export function TaskRunStatusIcon({
       return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "EXPIRED":
       return <TrashIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "TIMED_OUT":
+      return <TimedOutIcon className={cn(runStatusClassNameColor(status), className)} />;
 
     default: {
       assertNever(status);
@@ -174,6 +181,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
       return "text-error";
     case "CRASHED":
       return "text-error";
+    case "TIMED_OUT":
+      return "text-error";
     default: {
       assertNever(status);
     }
@@ -210,6 +219,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
       return "Crashed";
     case "EXPIRED":
       return "Expired";
+    case "TIMED_OUT":
+      return "Timed out";
     default: {
       assertNever(status);
     }

diff --git a/apps/webapp/app/components/runs/v3/TaskRunsTable.tsx b/apps/webapp/app/components/runs/v3/TaskRunsTable.tsx
@@ -9,11 +9,13 @@ import { BeakerIcon, BookOpenIcon, CheckIcon } from "@heroicons/react/24/solid";
 import { useLocation } from "@remix-run/react";
 import { formatDuration, formatDurationMilliseconds } from "@trigger.dev/core/v3";
 import { useCallback, useRef } from "react";
+import { Badge } from "~/components/primitives/Badge";
 import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { Checkbox } from "~/components/primitives/Checkbox";
 import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { Header3 } from "~/components/primitives/Headers";
 import { useSelectedItems } from "~/components/primitives/SelectedItemsProvider";
+import { SimpleTooltip } from "~/components/primitives/Tooltip";
 import { useEnvironments } from "~/hooks/useEnvironments";
 import { useFeatures } from "~/hooks/useFeatures";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -39,9 +41,12 @@ import {
 import { CancelRunDialog } from "./CancelRunDialog";
 import { LiveTimer } from "./LiveTimer";
 import { ReplayRunDialog } from "./ReplayRunDialog";
-import { TaskRunStatusCombo } from "./TaskRunStatus";
 import { RunTag } from "./RunTag";
-import { Badge } from "~/components/primitives/Badge";
+import {
+  descriptionForTaskRunStatus,
+  filterableTaskRunStatuses,
+  TaskRunStatusCombo,
+} from "./TaskRunStatus";
 
 type RunsTableProps = {
   total: number;
@@ -126,7 +131,27 @@ export function TaskRunsTable({
           <TableHeaderCell>Env</TableHeaderCell>
           <TableHeaderCell>Task</TableHeaderCell>
           <TableHeaderCell>Version</TableHeaderCell>
-          <TableHeaderCell>Status</TableHeaderCell>
+          <TableHeaderCell
+            tooltip={
+              <div className="flex flex-col divide-y divide-grid-dimmed">
+                {filterableTaskRunStatuses.map((status) => (
+                  <div
+                    key={status}
+                    className="grid grid-cols-[8rem_1fr] gap-x-2 py-2 first:pt-1 last:pb-1"
+                  >
+                    <div className="mb-0.5 flex items-center gap-1.5 whitespace-nowrap">
+                      <TaskRunStatusCombo status={status} />
+                    </div>
+                    <Paragraph variant="extra-small" className="!text-wrap text-text-dimmed">
+                      {descriptionForTaskRunStatus(status)}
+                    </Paragraph>
+                  </div>
+                ))}
+              </div>
+            }
+          >
+            Status
+          </TableHeaderCell>
           <TableHeaderCell>Started</TableHeaderCell>
           <TableHeaderCell
             colSpan={3}
@@ -287,7 +312,11 @@ export function TaskRunsTable({
                 </TableCell>
                 <TableCell to={path}>{run.version ?? "–"}</TableCell>
                 <TableCell to={path}>
-                  <TaskRunStatusCombo status={run.status} />
+                  <SimpleTooltip
+                    content={descriptionForTaskRunStatus(run.status)}
+                    disableHoverableContent
+                    button={<TaskRunStatusCombo status={run.status} />}
+                  />
                 </TableCell>
                 <TableCell to={path}>
                   {run.startedAt ? <DateTime date={run.startedAt} /> : "–"}

diff --git a/apps/webapp/app/database-types.ts b/apps/webapp/app/database-types.ts
@@ -42,6 +42,7 @@ export const TaskRunStatus = {
   CRASHED: "CRASHED",
   DELAYED: "DELAYED",
   EXPIRED: "EXPIRED",
+  TIMED_OUT: "TIMED_OUT",
 } as const satisfies Record<TaskRunStatusType, TaskRunStatusType>;
 
 export const JobRunStatus = {

diff --git a/apps/webapp/app/models/taskRun.server.ts b/apps/webapp/app/models/taskRun.server.ts
@@ -119,6 +119,7 @@ export function batchTaskRunItemStatusForRunStatus(
     case TaskRunStatus.SYSTEM_FAILURE:
     case TaskRunStatus.CRASHED:
     case TaskRunStatus.EXPIRED:
+    case TaskRunStatus.TIMED_OUT:
       return BatchTaskRunItemStatus.FAILED;
     case TaskRunStatus.PENDING:
     case TaskRunStatus.WAITING_FOR_DEPLOY:

diff --git a/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts
@@ -270,6 +270,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       case "EXPIRED": {
         return "EXPIRED";
       }
+      case "TIMED_OUT": {
+        return "TIMED_OUT";
+      }
       default: {
         assertNever(status);
       }

diff --git a/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts
@@ -310,6 +310,9 @@ export class ApiRunListPresenter extends BasePresenter {
       case "EXPIRED": {
         return "EXPIRED";
       }
+      case "TIMED_OUT": {
+        return "TIMED_OUT";
+      }
       default: {
         assertNever(status);
       }

diff --git a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts
@@ -9,6 +9,7 @@ import { eventRepository } from "~/v3/eventRepository.server";
 import { machinePresetFromName } from "~/v3/machinePresets.server";
 import { FINAL_ATTEMPT_STATUSES, isFinalRunStatus } from "~/v3/taskStatus";
 import { BasePresenter } from "./basePresenter.server";
+import { getMaxDuration } from "~/v3/utils/maxDuration";
 
 type Result = Awaited<ReturnType<SpanPresenter["call"]>>;
 export type Span = NonNullable<NonNullable<Result>["span"]>;
@@ -69,6 +70,7 @@ export class SpanPresenter extends BasePresenter {
         taskIdentifier: true,
         friendlyId: true,
         isTest: true,
+        maxDurationInSeconds: true,
         tags: {
           select: {
             name: true,
@@ -229,6 +231,7 @@ export class SpanPresenter extends BasePresenter {
         baseCostInCents: run.baseCostInCents,
         maxAttempts: run.maxAttempts ?? undefined,
         version: run.lockedToVersion?.version,
+        maxDuration: run.maxDurationInSeconds ?? undefined,
       },
       queue: {
         name: run.queue,
@@ -307,6 +310,7 @@ export class SpanPresenter extends BasePresenter {
       },
       context: JSON.stringify(context, null, 2),
       metadata,
+      maxDurationInSeconds: getMaxDuration(run.maxDurationInSeconds),
     };
   }
 

diff --git a/...rgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx b/...rgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx
@@ -677,7 +677,12 @@ function RunBody({
                     )}
                   </Property.Value>
                 </Property.Item>
-
+                <Property.Item>
+                  <Property.Label>Max duration</Property.Label>
+                  <Property.Value>
+                    {run.maxDurationInSeconds ? `${run.maxDurationInSeconds}s` : "–"}
+                  </Property.Value>
+                </Property.Item>
                 <Property.Item>
                   <Property.Label>Run invocation cost</Property.Label>
                   <Property.Value>

diff --git a/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/devQueueConsumer.server.ts
@@ -24,6 +24,7 @@ import {
   tracer,
 } from "../tracer.server";
 import { DevSubscriber, devPubSub } from "./devPubSub.server";
+import { getMaxDuration } from "../utils/maxDuration";
 
 const MessageBody = z.discriminatedUnion("type", [
   z.object({
@@ -378,6 +379,10 @@ export class DevQueueConsumer {
         status: "EXECUTING",
         lockedToVersionId: backgroundWorker.id,
         startedAt: existingTaskRun.startedAt ?? new Date(),
+        maxDurationInSeconds: getMaxDuration(
+          existingTaskRun.maxDurationInSeconds,
+          backgroundTask.maxDurationInSeconds
+        ),
       },
       include: {
         attempts: {

diff --git a/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts b/apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts
@@ -44,6 +44,7 @@ import { EnvironmentVariable } from "../environmentVariables/repository";
 import { machinePresetFromConfig } from "../machinePresets.server";
 import { env } from "~/env.server";
 import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
+import { getMaxDuration } from "../utils/maxDuration";
 
 const WithTraceContext = z.object({
   traceparent: z.string().optional(),
@@ -403,6 +404,10 @@ export class SharedQueueConsumer {
             startedAt: existingTaskRun.startedAt ?? new Date(),
             baseCostInCents: env.CENTS_PER_RUN,
             machinePreset: machinePresetFromConfig(backgroundTask.machineConfig ?? {}).name,
+            maxDurationInSeconds: getMaxDuration(
+              existingTaskRun.maxDurationInSeconds,
+              backgroundTask.maxDurationInSeconds
+            ),
           },
           include: {
             runtimeEnvironment: true,
@@ -1067,6 +1072,7 @@ class SharedQueueTasks {
         costInCents: taskRun.costInCents,
         baseCostInCents: taskRun.baseCostInCents,
         metadata,
+        maxDuration: taskRun.maxDurationInSeconds ?? undefined,
       },
       queue: {
         id: queue.friendlyId,

diff --git a/apps/webapp/app/v3/requeueTaskRun.server.ts b/apps/webapp/app/v3/requeueTaskRun.server.ts
@@ -70,6 +70,7 @@ export class RequeueTaskRunService extends BaseService {
       case "COMPLETED_WITH_ERRORS":
       case "COMPLETED_SUCCESSFULLY":
       case "EXPIRED":
+      case "TIMED_OUT":
       case "CANCELED": {
         logger.debug("[RequeueTaskRunService] Task run is completed", { taskRun });
 

diff --git a/apps/webapp/app/v3/services/completeAttempt.server.ts b/apps/webapp/app/v3/services/completeAttempt.server.ts
@@ -363,10 +363,16 @@ export class CompleteAttemptService extends BaseService {
           },
         });
 
+        const status =
+          sanitizedError.type === "INTERNAL_ERROR" &&
+          sanitizedError.code === "MAX_DURATION_EXCEEDED"
+            ? "TIMED_OUT"
+            : "COMPLETED_WITH_ERRORS";
+
         const finalizeService = new FinalizeTaskRunService();
         await finalizeService.call({
           id: taskRunAttempt.taskRunId,
-          status: "COMPLETED_WITH_ERRORS",
+          status,
           completedAt: new Date(),
         });
       }

diff --git a/apps/webapp/app/v3/services/createBackgroundWorker.server.ts b/apps/webapp/app/v3/services/createBackgroundWorker.server.ts
@@ -15,6 +15,7 @@ import { projectPubSub } from "./projectPubSub.server";
 import { RegisterNextTaskScheduleInstanceService } from "./registerNextTaskScheduleInstance.server";
 import cronstrue from "cronstrue";
 import { CheckScheduleService } from "./checkSchedule.server";
+import { clampMaxDuration } from "../utils/maxDuration";
 
 export class CreateBackgroundWorkerService extends BaseService {
   public async call(
@@ -156,6 +157,7 @@ export async function createBackgroundTasks(
           machineConfig: task.machine,
           triggerSource: task.triggerSource === "schedule" ? "SCHEDULED" : "STANDARD",
           fileId: tasksToBackgroundFiles?.get(task.id) ?? null,
+          maxDurationInSeconds: task.maxDuration ? clampMaxDuration(task.maxDuration) : null,
         },
       });
 

diff --git a/apps/webapp/app/v3/services/createTaskRunAttempt.server.ts b/apps/webapp/app/v3/services/createTaskRunAttempt.server.ts
@@ -196,6 +196,7 @@ export class CreateTaskRunAttemptService extends BaseService {
           maxAttempts: taskRun.maxAttempts ?? undefined,
           version: taskRun.lockedBy.worker.version,
           metadata,
+          maxDuration: taskRun.maxDurationInSeconds ?? undefined,
         },
         queue: {
           id: queue.friendlyId,

diff --git a/apps/webapp/app/v3/services/triggerTask.server.ts b/apps/webapp/app/v3/services/triggerTask.server.ts
@@ -23,6 +23,7 @@ import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.ser
 import { handleMetadataPacket } from "~/utils/packets";
 import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
 import { guardQueueSizeLimitsForEnv } from "../queueSizeLimits.server";
+import { clampMaxDuration } from "../utils/maxDuration";
 
 export type TriggerTaskServiceOptions = {
   idempotencyKey?: string;
@@ -373,6 +374,9 @@ export class TriggerTaskService extends BaseService {
                   metadataType: metadataPacket?.dataType,
                   seedMetadata: metadataPacket?.data,
                   seedMetadataType: metadataPacket?.dataType,
+                  maxDurationInSeconds: body.options?.maxDuration
+                    ? clampMaxDuration(body.options.maxDuration)
+                    : undefined,
                 },
               });
 

diff --git a/apps/webapp/app/v3/taskStatus.ts b/apps/webapp/app/v3/taskStatus.ts
@@ -41,6 +41,7 @@ export const FINAL_RUN_STATUSES = [
   "SYSTEM_FAILURE",
   "EXPIRED",
   "CRASHED",
+  "TIMED_OUT",
 ] satisfies TaskRunStatus[];
 
 export type FINAL_RUN_STATUSES = (typeof FINAL_RUN_STATUSES)[number];
@@ -96,6 +97,7 @@ export const FAILED_RUN_STATUSES = [
   "COMPLETED_WITH_ERRORS",
   "SYSTEM_FAILURE",
   "CRASHED",
+  "TIMED_OUT",
 ] satisfies TaskRunStatus[];
 
 export function isFailedRunStatus(status: TaskRunStatus): boolean {