From 641743d67e15a2c5613faa74c7b993e0b28dcd18 Mon Sep 17 00:00:00 2001 From: Jacob Snarr Date: Wed, 18 Dec 2024 11:19:39 -0500 Subject: [PATCH 1/4] feat(sdk-go): Handle LHTaskExceptions, Errors, and Panics (#1210) --- .../05-task-worker-development.md | 2 +- sdk-go/littlehorse/lh_errors.go | 9 +++ sdk-go/littlehorse/task_worker_internal.go | 62 +++++++++++++++++-- 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 sdk-go/littlehorse/lh_errors.go diff --git a/docs/docs/05-developer-guide/05-task-worker-development.md b/docs/docs/05-developer-guide/05-task-worker-development.md index c1ee6ec8d..440fb30f7 100644 --- a/docs/docs/05-developer-guide/05-task-worker-development.md +++ b/docs/docs/05-developer-guide/05-task-worker-development.md @@ -178,7 +178,7 @@ The Go SDK currently (as of `0.11.0`) does not yet support throwing `LHTaskExcep ```python -from littlehorse.exceptions import LHTaskExceptio +from littlehorse.exceptions import LHTaskException async def ship_item(item_sku: str) -> str: if is_out_of_stock(): diff --git a/sdk-go/littlehorse/lh_errors.go b/sdk-go/littlehorse/lh_errors.go new file mode 100644 index 000000000..4dd9610a0 --- /dev/null +++ b/sdk-go/littlehorse/lh_errors.go @@ -0,0 +1,9 @@ +package littlehorse + +import "github.com/littlehorse-enterprises/littlehorse/sdk-go/lhproto" + +type LHTaskException struct { + Name string + Message string + Content *lhproto.VariableValue +} diff --git a/sdk-go/littlehorse/task_worker_internal.go b/sdk-go/littlehorse/task_worker_internal.go index a928b9879..c2591bf18 100644 --- a/sdk-go/littlehorse/task_worker_internal.go +++ b/sdk-go/littlehorse/task_worker_internal.go @@ -5,6 +5,7 @@ import ( "github.com/littlehorse-enterprises/littlehorse/sdk-go/lhproto" "log" "reflect" + "runtime/debug" "strconv" "sync" "time" @@ -311,6 +312,7 @@ func (m *serverConnectionManager) submitTaskForExecution(task *lhproto.Scheduled } func (m *serverConnectionManager) doTask(taskToExec *taskExecutionInfo) { + defer m.recoverFromPanic(taskToExec) taskResult := m.doTaskHelper(taskToExec.task) _, err := (*taskToExec.specificStub).ReportTask(context.Background(), taskResult) if err != nil { @@ -318,6 +320,34 @@ func (m *serverConnectionManager) doTask(taskToExec *taskExecutionInfo) { } } +func (m *serverConnectionManager) recoverFromPanic(taskToExec *taskExecutionInfo) { + if v := recover(); v != nil { + varVal, _ := InterfaceToVarVal(v) + taskResult := &lhproto.ReportTaskRun{ + TaskRunId: taskToExec.task.TaskRunId, + Time: timestamppb.Now(), + Status: lhproto.TaskStatus(lhproto.LHStatus_ERROR), + LogOutput: &lhproto.VariableValue{ + Value: &lhproto.VariableValue_Str{ + Str: string(debug.Stack()), + }, + }, + AttemptNumber: taskToExec.task.AttemptNumber, + Result: &lhproto.ReportTaskRun_Error{ + Error: &lhproto.LHTaskError{ + Type: lhproto.LHErrorType_TASK_FAILURE, + Message: "Task Worker Panic: " + varVal.GetStr(), + }, + }, + } + _, err := (*taskToExec.specificStub).ReportTask(context.Background(), taskResult) + if err != nil { + log.Default().Print(err) + m.retryReportTask(context.Background(), taskResult, TOTAL_RETRIES) + } + } +} + func (m *serverConnectionManager) retryReportTask(ctx context.Context, taskResult *lhproto.ReportTaskRun, retries int) { log.Println("Retrying reportTask rpc on wfRun {}", taskResult.TaskRunId.WfRunId) @@ -399,16 +429,40 @@ func (m *serverConnectionManager) doTaskHelper(task *lhproto.ScheduledTask) *lhp errorReflect := invocationResults[len(invocationResults)-1] if errorReflect.Interface() != nil { - errorVarVal, err := InterfaceToVarVal(errorReflect.Interface()) - if err != nil { - log.Println("WARN: was unable to serialize error") + // Check if the error is an LHTaskException + if lhtErr, ok := errorReflect.Interface().(*LHTaskException); ok { + taskResult.Result = &lhproto.ReportTaskRun_Exception{ + Exception: &lhproto.LHTaskException{ + Name: lhtErr.Name, + Message: lhtErr.Message, + Content: lhtErr.Content, + }, + } } else { - taskResult.LogOutput = errorVarVal + // Otherwise, try to interpret the error + if err, ok := errorReflect.Interface().(error); ok { + taskResult.Result = &lhproto.ReportTaskRun_Error{ + Error: &lhproto.LHTaskError{ + Type: lhproto.LHErrorType_TASK_FAILURE, + Message: err.Error(), + }, + } + } else { + // If the error returned by the taskMethod does not match the error interface + taskResult.Result = &lhproto.ReportTaskRun_Error{ + Error: &lhproto.LHTaskError{ + Type: lhproto.LHErrorType_TASK_FAILURE, + Message: "Task Method error serialization failed.", + }, + } + } } + taskResult.Status = lhproto.TaskStatus_TASK_FAILED } } + taskResult.AttemptNumber = task.AttemptNumber taskResult.Time = timestamppb.Now() return taskResult From 8a82d0a3841c6f477fc5bbb2acbd29df72666925 Mon Sep 17 00:00:00 2001 From: KarlaCarvajal Date: Wed, 18 Dec 2024 14:54:44 -0600 Subject: [PATCH 2/4] build(sdk-dotnet): add dotnet job to release pipeline (#1214) * Add release job to publish sdk-dotnet package in nuget.org * Add test dotnet-sdk job Co-authored-by: Saul --- .github/workflows/release.yml | 26 +++++++++++++++++++ .github/workflows/tests.yml | 15 +++++++++++ .../LittleHorse.Sdk/LittleHorse.Sdk.csproj | 4 +-- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5edd5ac1c..ccc88ff82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -146,6 +146,31 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} run: npm publish --access public + sdk-dotnet: + runs-on: ubuntu-latest + needs: + - publish-docker + - prepare + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '6' + - name: Bump version + env: + TAG: ${{ needs.prepare.outputs.tag }} + run: sed -i "s/.*<\/PackageVersion>/${TAG}<\/PackageVersion>/g" sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj + - name: Build Project + run: dotnet build sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj + - name: Create Package + run: dotnet pack --configuration Release sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj + - name: Publish Package to nuget.org + run: dotnet nuget push sdk-dotnet/LittleHorse.Sdk/bin/Release/*.nupkg -k $NUGET_AUTH_TOKEN -s https://api.nuget.org/v3/index.json + env: + NUGET_AUTH_TOKEN: ${{ secrets.NUGET_TOKEN }} + lhctl: runs-on: ubuntu-latest steps: @@ -174,6 +199,7 @@ jobs: - sdk-java - sdk-python - sdk-js + - sdk-dotnet runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e523db727..7e834f9dd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -133,3 +133,18 @@ jobs: run: | npm ci npm run test + + tests-sdk-dotnet: + if: ${{ !contains(github.event.head_commit.message, '[skip main]') }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '6' + - name: Build Project + run: dotnet build sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj + - name: Test Dotnet + run: dotnet test sdk-dotnet/LittleHorse.Sdk.Tests \ No newline at end of file diff --git a/sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj b/sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj index 7b89de2a0..1f3e09655 100644 --- a/sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj +++ b/sdk-dotnet/LittleHorse.Sdk/LittleHorse.Sdk.csproj @@ -4,9 +4,7 @@ net6.0 enable enable - 0.5.8 - 0.5.8 - 0.5.8-alpha + 0.0.0 LittleHorse Enterprises LLC LittleHorse Enterprises LLC LittleHorseSDK From 3807fad6e0e491d02e8b77c75a304daebee9f0b4 Mon Sep 17 00:00:00 2001 From: "Bryson G." <114206517+bryson-g@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:09:34 -0800 Subject: [PATCH 3/4] feat(dashboard): display scheduledWfRuns (#1202) --- dashboard/package-lock.json | 53 ++++++++ dashboard/package.json | 2 + .../[...props]/actions/ScheduleWfRun.ts | 19 --- .../[...props]/actions/getScheduleWfSpec.ts | 24 +++- .../[...props]/components/ScheduledWfRuns.tsx | 124 ++++++++++++++++++ .../wfSpec/[...props]/components/WfRuns.tsx | 15 +-- .../[...props]/components/WfRunsHeader.tsx | 2 - .../wfSpec/[...props]/components/WfSpec.tsx | 20 ++- .../(diagram)/wfSpec/[...props]/page.tsx | 3 +- .../[tenantId]/components/LinkWithTenant.tsx | 9 +- .../[tenantId]/components/SelectionLink.tsx | 32 +++++ .../components/tables/WfSpecTable.tsx | 11 +- dashboard/src/app/constants.ts | 14 ++ dashboard/src/app/utils/getCronTimeWindow.ts | 18 +++ dashboard/src/components/ui/tabs.tsx | 55 ++++++++ 15 files changed, 348 insertions(+), 53 deletions(-) delete mode 100644 dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/ScheduleWfRun.ts create mode 100644 dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/ScheduledWfRuns.tsx create mode 100644 dashboard/src/app/[tenantId]/components/SelectionLink.tsx create mode 100644 dashboard/src/app/utils/getCronTimeWindow.ts create mode 100644 dashboard/src/components/ui/tabs.tsx diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 9ed39d2d5..e760647f4 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -18,10 +18,12 @@ "@radix-ui/react-separator": "^1.1.0", "@radix-ui/react-slot": "^1.1.0", "@radix-ui/react-switch": "^1.1.1", + "@radix-ui/react-tabs": "^1.1.1", "@tanstack/react-query": "^5.37.1", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "cmdk": "^1.0.0", + "cron-parser": "^4.9.0", "dagre": "^0.8.5", "littlehorse-client": "file://../sdk-js", "lucide-react": "^0.379.0", @@ -3544,6 +3546,36 @@ } } }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.1.tgz", + "integrity": "sha512-3GBUDmP2DvzmtYLMsHmpA1GtR46ZDZ+OreXM/N+kkQJOPIgytFWWTfDQmBQKBvaFS0Vno0FktdbVzN28KGrMdw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.0", + "@radix-ui/react-context": "1.1.1", + "@radix-ui/react-direction": "1.1.0", + "@radix-ui/react-id": "1.1.0", + "@radix-ui/react-presence": "1.1.1", + "@radix-ui/react-primitive": "2.0.0", + "@radix-ui/react-roving-focus": "1.1.0", + "@radix-ui/react-use-controllable-state": "1.1.0" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, "node_modules/@radix-ui/react-use-callback-ref": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz", @@ -9485,6 +9517,18 @@ "devOptional": true, "license": "MIT" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -17323,6 +17367,15 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0" } }, + "node_modules/luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index 8ccf848b5..0a989b96b 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -21,10 +21,12 @@ "@radix-ui/react-separator": "^1.1.0", "@radix-ui/react-slot": "^1.1.0", "@radix-ui/react-switch": "^1.1.1", + "@radix-ui/react-tabs": "^1.1.1", "@tanstack/react-query": "^5.37.1", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "cmdk": "^1.0.0", + "cron-parser": "^4.9.0", "dagre": "^0.8.5", "littlehorse-client": "file://../sdk-js", "lucide-react": "^0.379.0", diff --git a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/ScheduleWfRun.ts b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/ScheduleWfRun.ts deleted file mode 100644 index cd2def909..000000000 --- a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/ScheduleWfRun.ts +++ /dev/null @@ -1,19 +0,0 @@ -'use server' -import { lhClient } from '@/app/lhClient' -import { WithTenant } from '@/types' -import { ScheduleWfRequest } from 'littlehorse-client/proto' -import { ScheduledWfRun } from '../../../../../../../../sdk-js/dist/proto/scheduled_wf_run' - -export const ScheduleWfRun = async ({ - wfSpecName, - tenantId, - majorVersion, - revision, - parentWfRunId, - id, - variables, - cronExpression, -}: ScheduleWfRequest & WithTenant): Promise => { - const client = await lhClient({ tenantId }) - return client.scheduleWf({ wfSpecName, majorVersion, revision, parentWfRunId, id, variables, cronExpression }) -} diff --git a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/getScheduleWfSpec.ts b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/getScheduleWfSpec.ts index 347204681..aaceb50f3 100644 --- a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/getScheduleWfSpec.ts +++ b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/actions/getScheduleWfSpec.ts @@ -1,20 +1,30 @@ 'use server' + import { lhClient } from '@/app/lhClient' import { WithTenant } from '@/types' -import { ScheduledWfRunIdList } from 'littlehorse-client/proto' +import { ScheduledWfRun } from 'littlehorse-client/proto' type GetWfSpecProps = { name: string version: string } & WithTenant -export const getScheduleWfSpec = async ({ name, version, tenantId }: GetWfSpecProps): Promise => { +export const getScheduleWfSpec = async ({ name, version, tenantId }: GetWfSpecProps): Promise => { const client = await lhClient({ tenantId }) const [majorVersion, revision] = version.split('.') - return client.searchScheduledWfRun({ - wfSpecName: name, - majorVersion: parseInt(majorVersion) || 0, - revision: parseInt(revision) | 0, - }) + + return Promise.all( + ( + await client.searchScheduledWfRun({ + wfSpecName: name, + majorVersion: parseInt(majorVersion) || 0, + revision: parseInt(revision) || 0, + }) + ).results.map(async scheduledWfRun => { + return await client.getScheduledWfRun({ + id: scheduledWfRun.id, + }) + }) + ) } diff --git a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/ScheduledWfRuns.tsx b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/ScheduledWfRuns.tsx new file mode 100644 index 000000000..17c3355d4 --- /dev/null +++ b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/ScheduledWfRuns.tsx @@ -0,0 +1,124 @@ +'use client' + +import { ScheduledWfRunIdList, WfSpec } from 'littlehorse-client/proto' +import { getScheduleWfSpec } from '../actions/getScheduleWfSpec' +import { SelectionLink } from '@/app/[tenantId]/components/SelectionLink' +import { ScheduledWfRun } from 'littlehorse-client/proto' +import { FUTURE_TIME_RANGES, SEARCH_DEFAULT_LIMIT, TimeRange } from '@/app/constants' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' +import { ClockIcon } from 'lucide-react' +import { useEffect, useState, useMemo } from 'react' +import { getCronTimeWindow } from '@/app/utils/getCronTimeWindow' +import { parseExpression } from 'cron-parser' +import { utcToLocalDateTime } from '@/app/utils' +import { SearchVariableDialog } from './SearchVariableDialog' +import { SearchFooter } from '@/app/[tenantId]/components/SearchFooter' +import { useParams, useSearchParams } from 'next/navigation' +import { RefreshCwIcon } from 'lucide-react' + +export const ScheduledWfRuns = (spec: WfSpec) => { + const [currentWindow, setWindow] = useState(-1) + const [isLoading, setIsLoading] = useState(true) + const [error, setError] = useState(null) + const [scheduledWfRuns, setScheduledWfRuns] = useState([]) + const tenantId = useParams().tenantId as string + + useEffect(() => { + let isMounted = true + + const fetchScheduledWfRuns = async () => { + try { + setIsLoading(true) + setError(null) + const runs = await getScheduleWfSpec({ + name: spec.id!.name, + version: spec.id!.majorVersion + '.' + spec.id!.revision, + tenantId: tenantId, + }) + if (isMounted) { + setScheduledWfRuns(runs) + } + } catch (err) { + if (isMounted) { + setError(err instanceof Error ? err : new Error('Failed to fetch scheduled runs')) + } + } finally { + if (isMounted) { + setIsLoading(false) + } + } + } + + fetchScheduledWfRuns() + + return () => { + isMounted = false + } + }, [spec.id, tenantId]) + + const filteredScheduledWfRuns = useMemo( + () => + scheduledWfRuns + .filter(scheduledWfRun => { + if (currentWindow === -1) return true + const timeWindow = getCronTimeWindow(scheduledWfRun.cronExpression) + return timeWindow && timeWindow <= currentWindow + }) + .sort((a, b) => { + const timeA = parseExpression(a.cronExpression).next().toDate().getTime() + const timeB = parseExpression(b.cronExpression).next().toDate().getTime() + return timeA - timeB + }), + [currentWindow, scheduledWfRuns] + ) + + if (isLoading) { + return ( +
+ +
+ ) + } + + if (error) { + return ( +
+

Error loading scheduled runs

+

{error.message}

+
+ ) + } + + return ( +
+
+ +
+
+ {filteredScheduledWfRuns.map(scheduledWfRun => ( + +

{scheduledWfRun.id?.id}

+
+ +

{utcToLocalDateTime(parseExpression(scheduledWfRun.cronExpression).next().toDate().toISOString())}

+
+ + ))} +
+
+ ) +} diff --git a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRuns.tsx b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRuns.tsx index 2099bd7b0..b20605f55 100644 --- a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRuns.tsx +++ b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRuns.tsx @@ -1,5 +1,4 @@ 'use client' -import LinkWithTenant from '@/app/[tenantId]/components/LinkWithTenant' import { SearchFooter } from '@/app/[tenantId]/components/SearchFooter' import { SEARCH_DEFAULT_LIMIT, TIME_RANGES, TimeRange } from '@/app/constants' import { concatWfRunIds } from '@/app/utils' @@ -10,6 +9,7 @@ import { useParams, useSearchParams } from 'next/navigation' import { FC, Fragment, useMemo, useState } from 'react' import { PaginatedWfRunIdList, searchWfRun } from '../actions/searchWfRun' import { WfRunsHeader } from './WfRunsHeader' +import { SelectionLink } from '@/app/[tenantId]/components/SelectionLink' export const WfRuns: FC = spec => { const searchParams = useSearchParams() @@ -57,18 +57,13 @@ export const WfRuns: FC = spec => { ) : ( -
+
{data?.pages.map((page, i) => ( {page.results.map(wfRunId => ( -
- - {wfRunId.id} - -
+ +

{wfRunId.id}

+ ))} ))} diff --git a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRunsHeader.tsx b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRunsHeader.tsx index 465f2e752..51f791acc 100644 --- a/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRunsHeader.tsx +++ b/dashboard/src/app/[tenantId]/(diagram)/wfSpec/[...props]/components/WfRunsHeader.tsx @@ -20,8 +20,6 @@ export const WfRunsHeader: FC = ({ spec, currentStatus, currentWindow, se return (
-

WfRun Search

-