diff --git a/packages/app-executor/bin/executor.ts b/packages/app-executor/bin/executor.ts index 80b9d7f78..bc4df28cd 100644 --- a/packages/app-executor/bin/executor.ts +++ b/packages/app-executor/bin/executor.ts @@ -5,9 +5,10 @@ import { NodeRegistration, plugins as rivetPlugins, registerBuiltInNodes, + NodeDatasetProvider, } from '@ironclad/rivet-node'; import * as Rivet from '@ironclad/rivet-core'; -import { RivetPluginInitializer } from '@ironclad/rivet-core'; +import { InMemoryDatasetProvider, RivetPluginInitializer } from '@ironclad/rivet-core'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import { P, match } from 'ts-pattern'; @@ -106,12 +107,16 @@ const rivetDebugger = startDebuggerServer({ } try { + console.dir({ currentDebuggerState }); + + const datasetProvider = new InMemoryDatasetProvider(currentDebuggerState.datsets ?? []); const processor = createProcessor(project, { graph: graphId, inputs, ...currentDebuggerState.settings!, remoteDebugger: rivetDebugger, registry, + datasetProvider, onTrace: (trace) => { console.log(trace); }, diff --git a/packages/app/src/components/DataStudio.tsx b/packages/app/src/components/DataStudio.tsx index 27e97bfc5..c77b4d060 100644 --- a/packages/app/src/components/DataStudio.tsx +++ b/packages/app/src/components/DataStudio.tsx @@ -1,13 +1,12 @@ -import { FC, useEffect, useState } from 'react'; +import { FC, useState } from 'react'; import { useRecoilState, useRecoilValue } from 'recoil'; import { overlayOpenState } from '../state/ui'; import { css } from '@emotion/react'; import { projectState } from '../state/savedGraphs'; import { ErrorBoundary } from 'react-error-boundary'; -import useIndexedDb from '../hooks/useIndexedDb'; import { selectedDatasetState } from '../state/dataStudio'; import { toast } from 'react-toastify'; -import { Dataset, DatasetId, DatasetMetadata, DatasetRow, getError, newId } from '@ironclad/rivet-core'; +import { DatasetId, DatasetMetadata, DatasetRow, getError, newId } from '@ironclad/rivet-core'; import Button from '@atlaskit/button'; import TextField from '@atlaskit/textfield'; import clsx from 'clsx'; @@ -27,7 +26,7 @@ export const DataStudioRenderer: FC = () => { if (openOverlay !== 'dataStudio') return null; return ( - + 'Failed to render Data Studio'}> setOpenOverlay(undefined)} /> ); diff --git a/packages/app/src/components/PromptDesigner.tsx b/packages/app/src/components/PromptDesigner.tsx index bb8e23bf4..579fbfdbb 100644 --- a/packages/app/src/components/PromptDesigner.tsx +++ b/packages/app/src/components/PromptDesigner.tsx @@ -51,6 +51,7 @@ import { toast } from 'react-toastify'; import { produce } from 'immer'; import { overlayOpenState } from '../state/ui'; import { BrowserDatasetProvider } from '../io/BrowserDatasetProvider'; +import { datasetProvider } from '../utils/globals'; const styles = css` position: fixed; @@ -987,7 +988,7 @@ async function runAdHocChat(messages: ChatMessage[], config: AdHocChatConfig) { createSubProcessor: undefined!, settings, nativeApi: new TauriNativeApi(), - datasetProvider: new BrowserDatasetProvider(), + datasetProvider, processId: nanoid() as ProcessId, executionCache: new Map(), externalFunctions: {}, @@ -1045,7 +1046,7 @@ function useRunTestGroup() { const outputs = await processor.processGraph( { nativeApi: new TauriNativeApi(), - datasetProvider: new BrowserDatasetProvider(), + datasetProvider, settings, }, { diff --git a/packages/app/src/hooks/useLocalExecutor.ts b/packages/app/src/hooks/useLocalExecutor.ts index 13608b1b4..ebbf2e316 100644 --- a/packages/app/src/hooks/useLocalExecutor.ts +++ b/packages/app/src/hooks/useLocalExecutor.ts @@ -25,6 +25,7 @@ import { fillMissingSettingsFromEnvironmentVariables } from '../utils/tauri'; import { trivetState } from '../state/trivet'; import { runTrivet } from '@ironclad/trivet'; import { BrowserDatasetProvider } from '../io/BrowserDatasetProvider'; +import { datasetProvider } from '../utils/globals'; export function useLocalExecutor() { const project = useRecoilValue(projectState); @@ -129,7 +130,7 @@ export function useLocalExecutor() { globalRivetNodeRegistry.getPlugins(), ), nativeApi: new TauriNativeApi(), - datasetProvider: new BrowserDatasetProvider(), + datasetProvider, }); } @@ -187,7 +188,7 @@ export function useLocalExecutor() { globalRivetNodeRegistry.getPlugins(), ), nativeApi: new TauriNativeApi(), - datasetProvider: new BrowserDatasetProvider(), + datasetProvider, }, inputs, ); diff --git a/packages/app/src/hooks/useRemoteExecutor.ts b/packages/app/src/hooks/useRemoteExecutor.ts index 5dc0c6128..046dfecf3 100644 --- a/packages/app/src/hooks/useRemoteExecutor.ts +++ b/packages/app/src/hooks/useRemoteExecutor.ts @@ -4,6 +4,7 @@ import { ProcessEvents, StringArrayDataValue, globalRivetNodeRegistry, + serializeDatasets, } from '@ironclad/rivet-core'; import { useCurrentExecution } from './useCurrentExecution'; import { useRecoilState, useRecoilValue, useSetRecoilState } from 'recoil'; @@ -21,6 +22,7 @@ import { userInputModalQuestionsState, userInputModalSubmitState } from '../stat import { pluginsState } from '../state/plugins'; import { entries } from '../../../core/src/utils/typeSafety'; import { selectedExecutorState } from '../state/execution'; +import { datasetProvider } from '../utils/globals'; // TODO: This allows us to retrieve the GraphOutputs from the remote debugger. // If the remote debugger events had a unique ID for each run, this would feel a lot less hacky. @@ -146,6 +148,7 @@ export function useRemoteExecutor() { savedSettings, globalRivetNodeRegistry.getPlugins(), ), + datasets: serializeDatasets(await datasetProvider.exportDatasetsForProject(project.metadata.id)), }); for (const [id, dataValue] of entries(projectData)) { @@ -208,6 +211,7 @@ export function useRemoteExecutor() { savedSettings, globalRivetNodeRegistry.getPlugins(), ), + datasets: await datasetProvider.exportDatasetsForProject(project.metadata.id), }); } diff --git a/packages/app/src/io/BrowserDatasetProvider.ts b/packages/app/src/io/BrowserDatasetProvider.ts index f615bce9b..34137b0d7 100644 --- a/packages/app/src/io/BrowserDatasetProvider.ts +++ b/packages/app/src/io/BrowserDatasetProvider.ts @@ -52,7 +52,7 @@ export class BrowserDatasetProvider implements DatasetProvider { }; cursorRequest.onsuccess = () => { const cursor = cursorRequest.result; - if (cursor) { + if (cursor?.value) { const dataset = cursor.value as DatasetMetadata; if (dataset.projectId === projectId) { metadata.push(dataset); @@ -127,6 +127,14 @@ export class BrowserDatasetProvider implements DatasetProvider { if (matchingDataset) { matchingDataset.meta = metadata; + } else { + this.#currentProjectDatasets.push({ + meta: metadata, + data: { + id: metadata.id, + rows: [], + }, + }); } // Sync the database diff --git a/packages/core/src/model/nodes/ReadDirectoryNode.ts b/packages/core/src/model/nodes/ReadDirectoryNode.ts index 5e74bf686..e082cbba3 100644 --- a/packages/core/src/model/nodes/ReadDirectoryNode.ts +++ b/packages/core/src/model/nodes/ReadDirectoryNode.ts @@ -147,6 +147,12 @@ export class ReadDirectoryNodeImpl extends NodeImpl { } async process(inputData: Inputs, context: InternalProcessContext): Promise { + const { nativeApi } = context; + + if (nativeApi == null) { + throw new Error('This node requires a native API to run.'); + } + const path = this.chartNode.data.usePathInput ? expectType(inputData['path' as PortId], 'string') : this.chartNode.data.path; @@ -181,7 +187,7 @@ export class ReadDirectoryNodeImpl extends NodeImpl { } try { - const files = await context.nativeApi.readdir(path, undefined, { + const files = await nativeApi.readdir(path, undefined, { recursive, includeDirectories, filterGlobs, diff --git a/packages/core/src/model/nodes/ReadFileNode.ts b/packages/core/src/model/nodes/ReadFileNode.ts index 5eaf5ec3c..c4d48551e 100644 --- a/packages/core/src/model/nodes/ReadFileNode.ts +++ b/packages/core/src/model/nodes/ReadFileNode.ts @@ -70,12 +70,18 @@ export class ReadFileNodeImpl extends NodeImpl { inputData: Record, context: InternalProcessContext, ): Promise> { + const { nativeApi } = context; + + if (nativeApi == null) { + throw new Error('This node requires a native API to run.'); + } + const path = this.chartNode.data.usePathInput ? expectType(inputData['path' as PortId], 'string') : this.chartNode.data.path; try { - const content = await context.nativeApi.readTextFile(path, undefined); + const content = await nativeApi.readTextFile(path, undefined); return { ['content' as PortId]: { type: 'string', value: content }, }; diff --git a/packages/node/src/api.ts b/packages/node/src/api.ts index d6233fbdd..d3b99abfd 100644 --- a/packages/node/src/api.ts +++ b/packages/node/src/api.ts @@ -1,5 +1,6 @@ import { DataValue, + DatasetProvider, ExternalFunction, GraphId, GraphProcessor, @@ -47,6 +48,7 @@ export type RunGraphOptions = { context?: Record; remoteDebugger?: RivetDebuggerServer; nativeApi?: NativeApi; + datasetProvider?: DatasetProvider; externalFunctions?: { [key: string]: ExternalFunction; }; @@ -193,6 +195,7 @@ export function createProcessor(project: Project, options: RunGraphOptions) { const outputs = await processor.processGraph( { nativeApi: options.nativeApi ?? new NodeNativeApi(), + datasetProvider: options.datasetProvider, settings: { openAiKey: options.openAiKey ?? '', openAiOrganization: options.openAiOrganization ?? '', diff --git a/packages/node/src/debugger.ts b/packages/node/src/debugger.ts index f71a1b6d9..65b791f21 100644 --- a/packages/node/src/debugger.ts +++ b/packages/node/src/debugger.ts @@ -9,6 +9,8 @@ import { NodeId, StringArrayDataValue, DataId, + deserializeDatasets, + CombinedDataset, } from '@ironclad/rivet-core'; import { match } from 'ts-pattern'; import Emittery from 'emittery'; @@ -32,6 +34,7 @@ export interface DebuggerEvents { export const currentDebuggerState = { uploadedProject: undefined as Project | undefined, settings: undefined as Settings | undefined, + datsets: [] as CombinedDataset[] | undefined, }; export type DynamicGraphRunOptions = { @@ -90,9 +93,14 @@ export function startDebuggerServer( }) .with({ type: 'set-dynamic-data' }, async () => { if (options.allowGraphUpload) { - const { project, settings } = message.data as { project: Project; settings: Settings }; + const { project, settings, datasets } = message.data as { + project: Project; + settings: Settings; + datasets: string; + }; currentDebuggerState.uploadedProject = project; currentDebuggerState.settings = settings; + currentDebuggerState.datsets = deserializeDatasets(datasets); } }) .otherwise(async () => { diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index 24cf27117..4f661876b 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -3,3 +3,4 @@ export * from '@ironclad/rivet-core'; export * from './native/NodeNativeApi.js'; export * from './api.js'; export * from './debugger.js'; +export * from './native/NodeDatasetProvider.js';