From b4b227087c3641f735cd54cfe00eda2768b0f53f Mon Sep 17 00:00:00 2001 From: Nico <1622112+nicarq@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:19:24 -0600 Subject: [PATCH] Pyodide IDBFS Sync (#588) * playing around * update * fix recursiveness * add tests and refactor * fix tests * unify message component, rearrange files and setup jest * remove jest from shinkai ui --------- Co-authored-by: paulclindo --- apps/shinkai-desktop/jest.config.ts | 11 + .../chat/components/message-list.tsx | 3 + .../components/chat/components/message.tsx | 19 +- .../chat/python-code-runner/error-render.tsx | 0 .../__tests__/usePyodideInstance.test.tsx | 122 ++++++ .../hooks/usePyodideInstance.ts | 43 +++ .../chat/python-code-runner/output-render.tsx | 0 .../python-code-runner-web-worker.ts | 56 +-- .../python-code-runner/python-code-runner.tsx | 358 ++++++++++++++++++ .../services/__mocks__/shinkai-message-ts.ts | 21 + .../__tests__/file-system-service.test.ts | 281 ++++++++++++++ .../services/__tests__/job-service.test.ts | 249 ++++++++++++ .../services/file-system-service.ts | 213 +++++++++++ .../services/job-service.ts | 175 +++++++++ .../chat/python-code-runner/stderr-render.tsx | 0 .../chat/python-code-runner/stdout-render.tsx | 0 .../components/tool-playground.tsx | 2 +- .../src/components/sheet/table-chat.tsx | 9 +- apps/shinkai-desktop/src/test-setup.ts | 22 ++ apps/shinkai-desktop/tsconfig.spec.json | 8 +- libs/shinkai-message-ts/src/api/methods.ts | 17 + .../src/api/vector-fs/types.ts | 6 + libs/shinkai-node-state/src/v2/constants.ts | 1 + .../src/v2/queries/getJobContents/index.ts | 15 + .../src/v2/queries/getJobContents/types.ts | 25 ++ .../getJobContents/useGetJobContents.ts | 17 + libs/shinkai-ui/src/components/chat/index.ts | 6 - .../src/components/chat/message-list.tsx | 14 +- .../src/components/chat/message.tsx | 14 +- .../python-code-runner/python-code-runner.tsx | 224 ----------- 30 files changed, 1618 insertions(+), 313 deletions(-) create mode 100644 apps/shinkai-desktop/jest.config.ts rename {libs/shinkai-ui => apps/shinkai-desktop}/src/components/chat/python-code-runner/error-render.tsx (100%) create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/__tests__/usePyodideInstance.test.tsx create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/usePyodideInstance.ts rename {libs/shinkai-ui => apps/shinkai-desktop}/src/components/chat/python-code-runner/output-render.tsx (100%) rename {libs/shinkai-ui => apps/shinkai-desktop}/src/components/chat/python-code-runner/python-code-runner-web-worker.ts (89%) create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner.tsx create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/services/__mocks__/shinkai-message-ts.ts create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/file-system-service.test.ts create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/job-service.test.ts create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/services/file-system-service.ts create mode 100644 apps/shinkai-desktop/src/components/chat/python-code-runner/services/job-service.ts rename {libs/shinkai-ui => apps/shinkai-desktop}/src/components/chat/python-code-runner/stderr-render.tsx (100%) rename {libs/shinkai-ui => apps/shinkai-desktop}/src/components/chat/python-code-runner/stdout-render.tsx (100%) create mode 100644 apps/shinkai-desktop/src/test-setup.ts create mode 100644 libs/shinkai-node-state/src/v2/queries/getJobContents/index.ts create mode 100644 libs/shinkai-node-state/src/v2/queries/getJobContents/types.ts create mode 100644 libs/shinkai-node-state/src/v2/queries/getJobContents/useGetJobContents.ts delete mode 100644 libs/shinkai-ui/src/components/chat/python-code-runner/python-code-runner.tsx diff --git a/apps/shinkai-desktop/jest.config.ts b/apps/shinkai-desktop/jest.config.ts new file mode 100644 index 000000000..0f9935c13 --- /dev/null +++ b/apps/shinkai-desktop/jest.config.ts @@ -0,0 +1,11 @@ +export default { + displayName: 'shinkai-desktop', + preset: '../../jest.preset.js', + transform: { + '^.+\\.[tj]sx?$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], + }, + moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'], + coverageDirectory: '../../coverage/apps/shinkai-desktop', + setupFilesAfterEnv: ['/src/test-setup.ts'], + testEnvironment: 'jsdom', +}; diff --git a/apps/shinkai-desktop/src/components/chat/components/message-list.tsx b/apps/shinkai-desktop/src/components/chat/components/message-list.tsx index 15ce5ec82..56f931d79 100644 --- a/apps/shinkai-desktop/src/components/chat/components/message-list.tsx +++ b/apps/shinkai-desktop/src/components/chat/components/message-list.tsx @@ -66,6 +66,7 @@ export const MessageList = ({ regenerateFirstMessage, disabledRetryAndEdit, messageExtra, + hidePythonExecution, }: { noMoreMessageLabel: string; isSuccess: boolean; @@ -85,6 +86,7 @@ export const MessageList = ({ lastMessageContent?: React.ReactNode; disabledRetryAndEdit?: boolean; messageExtra?: React.ReactNode; + hidePythonExecution?: boolean; }) => { const chatContainerRef = useRef(null); const previousChatHeightRef = useRef(0); @@ -302,6 +304,7 @@ export const MessageList = ({ ? handleFirstMessageRetry : handleRetryMessage } + hidePythonExecution={hidePythonExecution} key={`${message.messageId}::${messageIndex}`} message={message} messageId={message.messageId} diff --git a/apps/shinkai-desktop/src/components/chat/components/message.tsx b/apps/shinkai-desktop/src/components/chat/components/message.tsx index 7adf1d42c..d68c45035 100644 --- a/apps/shinkai-desktop/src/components/chat/components/message.tsx +++ b/apps/shinkai-desktop/src/components/chat/components/message.tsx @@ -4,6 +4,7 @@ import { ToolArgs, ToolStatusType, } from '@shinkai_network/shinkai-message-ts/api/general/types'; +import { extractJobIdFromInbox } from '@shinkai_network/shinkai-message-ts/utils'; import { FormattedMessage } from '@shinkai_network/shinkai-node-state/v2/queries/getChatConversation/types'; import { Accordion, @@ -22,7 +23,6 @@ import { Form, FormField, MarkdownText, - PythonCodeRunner, Tooltip, TooltipContent, TooltipPortal, @@ -50,9 +50,11 @@ import { useForm } from 'react-hook-form'; import { Link } from 'react-router-dom'; import { z } from 'zod'; +import { useAuth } from '../../../store/auth'; import { useOAuth } from '../../../store/oauth'; import { oauthUrlMatcherFromErrorMessage } from '../../../utils/oauth'; import { useChatStore } from '../context/chat-context'; +import { PythonCodeRunner } from '../python-code-runner/python-code-runner'; export const extractErrorPropertyOrContent = ( content: string, @@ -83,6 +85,7 @@ type MessageProps = { disabledEdit?: boolean; handleEditMessage?: (message: string) => void; messageExtra?: React.ReactNode; + hidePythonExecution?: boolean; }; const actionBar = { @@ -162,6 +165,7 @@ type EditMessageFormSchema = z.infer; export const MessageBase = ({ message, // messageId, + hidePythonExecution, isPending, handleRetryMessage, disabledRetry, @@ -211,6 +215,8 @@ export const MessageBase = ({ const { setOauthModalVisible } = useOAuth(); + const auth = useAuth((state) => state.auth); + return ( )} - {pythonCode && } + {pythonCode && !hidePythonExecution && ( + + )} {oauthUrl && (
diff --git a/libs/shinkai-ui/src/components/chat/python-code-runner/error-render.tsx b/apps/shinkai-desktop/src/components/chat/python-code-runner/error-render.tsx similarity index 100% rename from libs/shinkai-ui/src/components/chat/python-code-runner/error-render.tsx rename to apps/shinkai-desktop/src/components/chat/python-code-runner/error-render.tsx diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/__tests__/usePyodideInstance.test.tsx b/apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/__tests__/usePyodideInstance.test.tsx new file mode 100644 index 000000000..9253614ad --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/__tests__/usePyodideInstance.test.tsx @@ -0,0 +1,122 @@ +import { renderHook } from '@testing-library/react'; +import { loadPyodide, PyodideInterface } from 'pyodide'; + +import { usePyodideInstance } from '../usePyodideInstance'; + +// Mock pyodide +jest.mock('pyodide', () => ({ + loadPyodide: jest.fn(), +})); + +describe('usePyodideInstance', () => { + let mockPyodide: jest.Mocked; + + beforeEach(() => { + // Reset mocks + jest.clearAllMocks(); + + // Create mock Pyodide instance + mockPyodide = { + FS: { + mount: jest.fn(), + readdir: jest.fn(), + stat: jest.fn(), + isDir: jest.fn(), + readFile: jest.fn(), + writeFile: jest.fn(), + unlink: jest.fn(), + mkdir: jest.fn(), + rmdir: jest.fn(), + syncfs: jest.fn(), + filesystems: { + IDBFS: 'IDBFS', + }, + }, + } as unknown as jest.Mocked; + + // Mock loadPyodide to return our mock instance + (loadPyodide as jest.Mock).mockResolvedValue(mockPyodide); + }); + + it('should initialize Pyodide and file system service', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error populate + (populate: boolean, callback: (err: Error | null) => void) => + callback(null), + ); + + const { result } = renderHook(() => usePyodideInstance()); + + // Initially, both pyodide and fileSystemService should be null + expect(result.current.pyodide).toBeNull(); + expect(result.current.fileSystemService).toBeNull(); + + // Initialize + const { pyodide, fileSystemService } = + await result.current.initializePyodide(); + + // After initialization + expect(pyodide).toBe(mockPyodide); + expect(fileSystemService).toBeDefined(); + expect(loadPyodide).toHaveBeenCalledWith({ + indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.26.2/full/', + stdout: console.log, + stderr: console.error, + }); + expect(mockPyodide.FS.mount).toHaveBeenCalledWith( + 'IDBFS', + {}, + '/home/pyodide', + ); + expect(mockPyodide.FS.syncfs).toHaveBeenCalledWith( + true, + expect.any(Function), + ); + }); + + it('should reuse existing Pyodide instance', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error populate + (populate: boolean, callback: (err: Error | null) => void) => + callback(null), + ); + + const { result } = renderHook(() => usePyodideInstance()); + + // First initialization + const first = await result.current.initializePyodide(); + expect(loadPyodide).toHaveBeenCalledTimes(1); + + // Second initialization + const second = await result.current.initializePyodide(); + expect(loadPyodide).toHaveBeenCalledTimes(1); // Should not be called again + expect(second.pyodide).toBe(first.pyodide); + expect(second.fileSystemService).toBe(first.fileSystemService); + }); + + it('should handle initialization errors', async () => { + (loadPyodide as jest.Mock).mockRejectedValue( + new Error('Failed to load Pyodide'), + ); + + const { result } = renderHook(() => usePyodideInstance()); + + await expect(result.current.initializePyodide()).rejects.toThrow( + 'Failed to load Pyodide', + ); + }); + + it('should handle file system initialization errors', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error populate + (populate: boolean, callback: (err: Error | null) => void) => + callback(new Error('Sync failed')), + ); + + const { result } = renderHook(() => usePyodideInstance()); + + await expect(result.current.initializePyodide()).rejects.toThrow( + 'Sync failed', + ); + }); +}); diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/usePyodideInstance.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/usePyodideInstance.ts new file mode 100644 index 000000000..bc495c804 --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/hooks/usePyodideInstance.ts @@ -0,0 +1,43 @@ +import { loadPyodide, PyodideInterface } from 'pyodide'; +import { useCallback, useRef } from 'react'; + +import { IFileSystemService, PyodideFileSystemService } from '../services/file-system-service'; + +export function usePyodideInstance() { + const pyodideRef = useRef(null); + const fileSystemServiceRef = useRef(null); + + const initializePyodide = useCallback(async () => { + if (pyodideRef.current) { + console.log('Pyodide is already initialized.'); + return { pyodide: pyodideRef.current, fileSystemService: fileSystemServiceRef.current! }; + } + + console.time('initialize pyodide'); + const pyodide = await loadPyodide({ + indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.26.2/full/', + stdout: console.log, + stderr: console.error, + }); + console.log('Pyodide initialized'); + + pyodideRef.current = pyodide; + fileSystemServiceRef.current = new PyodideFileSystemService(pyodide); + + try { + await fileSystemServiceRef.current.initialize(); + } catch (error) { + console.error('Failed to initialize file system:', error); + throw error; + } + + console.timeEnd('initialize pyodide'); + return { pyodide, fileSystemService: fileSystemServiceRef.current }; + }, []); + + return { + pyodide: pyodideRef.current, + fileSystemService: fileSystemServiceRef.current, + initializePyodide, + }; +} \ No newline at end of file diff --git a/libs/shinkai-ui/src/components/chat/python-code-runner/output-render.tsx b/apps/shinkai-desktop/src/components/chat/python-code-runner/output-render.tsx similarity index 100% rename from libs/shinkai-ui/src/components/chat/python-code-runner/output-render.tsx rename to apps/shinkai-desktop/src/components/chat/python-code-runner/output-render.tsx diff --git a/libs/shinkai-ui/src/components/chat/python-code-runner/python-code-runner-web-worker.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner-web-worker.ts similarity index 89% rename from libs/shinkai-ui/src/components/chat/python-code-runner/python-code-runner-web-worker.ts rename to apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner-web-worker.ts index 757255760..4fb6180ec 100644 --- a/libs/shinkai-ui/src/components/chat/python-code-runner/python-code-runner-web-worker.ts +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner-web-worker.ts @@ -232,12 +232,12 @@ const fetchPage = ( method: 'GET' | 'POST', body: any = null, ): string => { - console.log('fetchPage called with url:', url); - console.log('fetchPage called with headers:', headers); - console.log('fetchPage called with method:', method); - if (body) { - console.log('fetchPage called with body:', body); - } + // console.log('fetchPage called with url:', url); + // console.log('fetchPage called with headers:', headers); + // console.log('fetchPage called with method:', method); + // if (body) { + // console.log('fetchPage called with body:', body); + // } const filteredHeaders: Record = {}; for (const [key, value] of Object.entries(headers.toJs())) { @@ -285,8 +285,6 @@ const fetchPage = ( sharedBuffer, }); - console.log('Posted message to main thread, waiting for response...'); - const textDecoder = new TextDecoder(); let result = ''; let moreChunks = true; @@ -297,8 +295,6 @@ const fetchPage = ( // This loop will block the thread until syncArray[0] changes } - console.log('Polling done with status: ', syncArray[0]); - if (syncArray[0] === -1) { const errorMessage = textDecoder.decode(dataArray); console.error('Error fetching page:', errorMessage); @@ -309,8 +305,6 @@ const fetchPage = ( const chunk = textDecoder.decode(dataArray).replace(/\0/g, '').trim(); result += chunk; - console.log(`Received chunk of length: ${chunk.length}`); - // Check if more chunks are needed if (syncArray[0] === 1) { moreChunks = false; // Success, all chunks received @@ -377,48 +371,10 @@ const initialize = async () => { console.timeEnd('initialize'); }; -// Function to print contents of a directory -function printDirectoryContents(dirPath: string) { - try { - const entries = pyodide.FS.readdir(dirPath); - const folders: Array = []; - const files: Array = []; - - entries.forEach((entry: string) => { - if (entry === '.' || entry === '..') return; - const path = `${dirPath}/${entry}`; - const stat = pyodide.FS.stat(path); - if (pyodide.FS.isDir(stat.mode)) { - folders.push(entry); - } else if (pyodide.FS.isFile(stat.mode)) { - files.push(entry); - } - }); - - console.log(`Contents of ${dirPath}:`); - console.log('Folders:', folders); - console.log('Files:', files); - } catch (error) { - console.error(`Error reading ${dirPath} directory:`, error); - } -} - // Function to synchronize the filesystem to IndexedDB const syncFilesystem = async (save = false) => { return new Promise((resolve, reject) => { pyodide.FS.syncfs(save, (err: any) => { - printDirectoryContents('/home/pyodide'); - - printDirectoryContents('/home/web_user'); - - // Print contents inside the /home directory - printDirectoryContents('/home'); - - printDirectoryContents('/new_mnt'); - - // Print contents inside the root directory - printDirectoryContents('/'); - if (err) { console.error('syncfs error:', err); reject(err); diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner.tsx b/apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner.tsx new file mode 100644 index 000000000..20d95b558 --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/python-code-runner.tsx @@ -0,0 +1,358 @@ +import { useTranslation } from '@shinkai_network/shinkai-i18n'; +import { addFileToJob } from '@shinkai_network/shinkai-message-ts/api/jobs/index'; +import { DirectoryContent } from '@shinkai_network/shinkai-message-ts/api/vector-fs/types'; +import { useGetDownloadFile } from '@shinkai_network/shinkai-node-state/v2/queries/getDownloadFile/useGetDownloadFile'; +import { useGetJobContents } from '@shinkai_network/shinkai-node-state/v2/queries/getJobContents/useGetJobContents'; +import { Button } from '@shinkai_network/shinkai-ui'; +import { useMutation, UseMutationOptions } from '@tanstack/react-query'; +import { invoke } from '@tauri-apps/api/core'; +import { AnimatePresence, motion } from 'framer-motion'; +import { useEffect, useState } from 'react'; + +import { usePyodideInstance } from './hooks/usePyodideInstance'; +import { OutputRender } from './output-render'; +import { RunResult } from './python-code-runner-web-worker'; +import PythonRunnerWorker from './python-code-runner-web-worker?worker'; +import { FileSystemEntry } from './services/file-system-service'; +import { JobService } from './services/job-service'; + +// Utility function to create a delay +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +type PythonCodeRunnerProps = { + code: string; + jobId: string; + nodeAddress: string; + token: string; +}; + +// Define more specific message types +type PageMessage = { + type: 'page'; + method: 'GET' | 'POST'; + meta: string; + headers: Record; + body?: string; + sharedBuffer: SharedArrayBuffer; +}; + +type RunDoneMessage = { + type: 'run-done'; + payload: RunResult; +}; + +type WorkerMessage = PageMessage | RunDoneMessage; + +// Type guard functions +function isPageMessage(message: WorkerMessage): message is PageMessage { + return message.type === 'page'; +} + +function isRunDoneMessage(message: WorkerMessage): message is RunDoneMessage { + return message.type === 'run-done'; +} + +export const usePythonRunnerRunMutation = ( + options?: UseMutationOptions, +) => { + const response = useMutation({ + mutationFn: async (params: { code: string }): Promise => { + const worker = new PythonRunnerWorker(); + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error('execution timed out')); + }, 120000); // 2 minutes + + worker.onmessage = async (event: { data: WorkerMessage }) => { + if (isPageMessage(event.data)) { + const { + method, + meta: url, + headers, + body, + sharedBuffer, + } = event.data; + console.log(`main thread> ${method.toLowerCase()}ing page`, url); + console.log('main thread> headers: ', headers); + + const syncArray = new Int32Array(sharedBuffer, 0, 1); + const dataArray = new Uint8Array(sharedBuffer, 4); + + const bufferSize = 512 * 1024; + const maxBufferSize = 100 * 1024 * 1024; + let success = false; + + while (bufferSize <= maxBufferSize && !success) { + try { + console.log( + `main thread> ${method.toLowerCase()}ing page`, + url, + ); + const response = await invoke<{ + status: number; + headers: Record; + body: string; + }>(method === 'GET' ? 'get_request' : 'post_request', { + url, + customHeaders: JSON.stringify(headers), + ...(method === 'POST' && { body: JSON.stringify(body) }), + }); + console.log( + `main thread> ${method.toLowerCase()} response`, + response, + ); + + if (response.status >= 200 && response.status < 300) { + const textEncoder = new TextEncoder(); + const encodedData = textEncoder.encode(response.body); + + console.log('Required buffer size:', encodedData.length); + + let offset = 0; + while (offset < encodedData.length) { + const chunkSize = Math.min( + dataArray.length, + encodedData.length - offset, + ); + dataArray.set( + encodedData.subarray(offset, offset + chunkSize), + ); + offset += chunkSize; + + syncArray[0] = 2; + console.log( + 'main thread> Notifying Atomics with chunk ready', + ); + Atomics.notify(syncArray, 0); + + while (syncArray[0] === 2) { + await delay(25); + } + } + + syncArray[0] = 1; + console.log('main thread> Notifying Atomics with success'); + Atomics.notify(syncArray, 0); + success = true; + } else { + throw new Error(`HTTP Error: ${response.status}`); + } + } catch (error) { + let errorMessage = 'Unknown error'; + if (error instanceof Error) { + errorMessage = error.message; + } + console.error( + `main thread> error using ${method.toLowerCase()} with page`, + errorMessage, + ); + + const textEncoder = new TextEncoder(); + const encodedError = textEncoder.encode(errorMessage); + + if (encodedError.length <= dataArray.length) { + dataArray.set(encodedError); + } else { + console.warn('Error message too long to fit in buffer'); + } + + console.log('main thread> Notifying Atomics with error'); + syncArray[0] = -1; + Atomics.notify(syncArray, 0); + + await delay(10); + reject( + new Error( + `Failed to ${method.toLowerCase()} page: ` + errorMessage, + ), + ); + return; + } + } + } else if (isRunDoneMessage(event.data)) { + clearTimeout(timeout); + console.log('main thread> worker event', event); + resolve(event.data.payload); + } + }; + + worker.onerror = (error: { message: string }) => { + console.log('worker error', error); + clearTimeout(timeout); + reject(new Error(`worker error: ${error.message}`)); + }; + + worker.postMessage({ type: 'run', payload: { code: params.code } }); + }).finally(() => { + worker.terminate(); + }); + }, + ...options, + onSuccess: (...onSuccessParameters) => { + if (options?.onSuccess) { + options.onSuccess(...onSuccessParameters); + } + }, + }); + return { ...response }; +}; + +// Define a function to transform DirectoryContent to FileSystemEntry +function transformToFileSystemEntry( + contents: DirectoryContent[], +): FileSystemEntry[] { + return contents.map((entry) => ({ + name: entry.name, + type: entry.is_directory ? 'directory' : 'file', + content: undefined, + contents: entry.is_directory + ? transformToFileSystemEntry(entry.children || []) + : undefined, + mtimeMs: new Date(entry.modified_time).getTime(), + })); +} + +export const PythonCodeRunner = ({ + code, + jobId, + nodeAddress, + token, +}: PythonCodeRunnerProps) => { + const i18n = useTranslation(); + // @ts-expect-error unused-vars + const [isSyncing, setIsSyncing] = useState(false); + // @ts-expect-error unused-vars + const [jobContents, setJobContents] = useState( + null, + ); + + const { + mutateAsync: run, + data: runResult, + isPending, + } = usePythonRunnerRunMutation(); + + const { mutateAsync: downloadFile } = useGetDownloadFile(); + + const { data: fetchedJobContents, refetch: refetchJobContents } = + useGetJobContents( + { + nodeAddress, + token, + jobId, + }, + { + enabled: false, + }, + ); + // @ts-expect-error unused-vars + const { pyodide, fileSystemService, initializePyodide } = + usePyodideInstance(); + + useEffect(() => { + if (fetchedJobContents) { + const transformedContents = + transformToFileSystemEntry(fetchedJobContents); + setJobContents(transformedContents); + } + }, [fetchedJobContents]); + + return ( +
+
+ +
+ + + {!isPending && runResult && ( + + + + )} + +
+ ); +}; diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__mocks__/shinkai-message-ts.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__mocks__/shinkai-message-ts.ts new file mode 100644 index 000000000..ea8bf9b4e --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__mocks__/shinkai-message-ts.ts @@ -0,0 +1,21 @@ +export interface AddFileToJobRequest { + job_id: string; + filename: string; + file: File; +} + +export interface AddFileToInboxResponse { + message: string; + filename: string; +} + +export interface DirectoryContent { + name: string; + path: string; + is_directory: boolean; + children: DirectoryContent[] | null; + created_time: string; + modified_time: string; + has_embeddings: boolean; + size: number; +} \ No newline at end of file diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/file-system-service.test.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/file-system-service.test.ts new file mode 100644 index 000000000..e7f97582f --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/file-system-service.test.ts @@ -0,0 +1,281 @@ +import { PyodideInterface } from 'pyodide'; + +import { PyodideFileSystemService } from '../file-system-service'; + +type FSCallback = (error: Error | null) => void; +type MockStats = { [key: string]: { mode: number } }; + +describe('PyodideFileSystemService', () => { + let mockPyodide: jest.Mocked; + let service: PyodideFileSystemService; + + beforeEach(() => { + // Create mock Pyodide instance + mockPyodide = { + FS: { + mount: jest.fn(), + readdir: jest.fn(), + stat: jest.fn(), + isDir: jest.fn(), + readFile: jest.fn(), + writeFile: jest.fn(), + unlink: jest.fn(), + mkdir: jest.fn(), + rmdir: jest.fn(), + syncfs: jest.fn(), + filesystems: { + IDBFS: 'IDBFS', + }, + }, + } as unknown as jest.Mocked; + + service = new PyodideFileSystemService(mockPyodide); + }); + + describe('initialize', () => { + it('should mount IDBFS and sync from IndexedDB', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error unused-vars + (populate: boolean, callback: FSCallback) => callback(null), + ); + + await service.initialize(); + + expect(mockPyodide.FS.mount).toHaveBeenCalledWith( + 'IDBFS', + {}, + '/home/pyodide', + ); + expect(mockPyodide.FS.syncfs).toHaveBeenCalledWith( + true, + expect.any(Function), + ); + }); + + it('should throw error if mounting fails', async () => { + mockPyodide.FS.mount.mockImplementation(() => { + throw new Error('Mount failed'); + }); + + await expect(service.initialize()).rejects.toThrow('Mount failed'); + }); + + it('should throw error if sync fails', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error unused-vars + (populate: boolean, callback: FSCallback) => + callback(new Error('Sync failed')), + ); + + await expect(service.initialize()).rejects.toThrow('Sync failed'); + }); + }); + + describe('readContents', () => { + it('should read directory contents correctly', () => { + const mockEntries = ['file1.txt', 'dir1', '.', '..', '.matplotlib']; + const mockStats: MockStats = { + 'file1.txt': { mode: 0o100644 }, // regular file + dir1: { mode: 0o040000 }, // directory + }; + + // Mock first readdir call for root directory + mockPyodide.FS.readdir.mockImplementation((path: string) => { + if (path === '/test') { + return mockEntries; + } + // Return empty directory for recursive calls + return ['.', '..']; + }); + + mockPyodide.FS.stat.mockImplementation((path: string) => { + const name = path.split('/').pop() || ''; + const stat = mockStats[name]; + if (!stat) { + throw new Error(`No mock stat for ${name}`); + } + return stat; + }); + mockPyodide.FS.isDir.mockImplementation( + (mode: number) => mode === 0o040000, + ); + mockPyodide.FS.readFile.mockReturnValue('file content'); + + const result = service.readContents('/test'); + + expect(result).toEqual([ + { + name: 'file1.txt', + type: 'file', + content: 'file content', + }, + { + name: 'dir1', + type: 'directory', + contents: [], // Empty array since we mock empty directory for recursive calls + }, + ]); + }); + + it('should handle errors and return null', () => { + mockPyodide.FS.readdir.mockImplementation(() => { + throw new Error('Read failed'); + }); + + const result = service.readContents('/test'); + + expect(result).toBeNull(); + }); + }); + + describe('writeFile', () => { + it('should write file content correctly', () => { + service.writeFile('/test/file.txt', 'content'); + + expect(mockPyodide.FS.writeFile).toHaveBeenCalledWith( + '/test/file.txt', + 'content', + { encoding: 'utf8' }, + ); + }); + + it('should throw error if write fails', () => { + mockPyodide.FS.writeFile.mockImplementation(() => { + throw new Error('Write failed'); + }); + + expect(() => service.writeFile('/test/file.txt', 'content')).toThrow( + 'Write failed', + ); + }); + }); + + describe('ensureDirectory', () => { + it('should create directory structure correctly', () => { + mockPyodide.FS.stat.mockImplementation(() => { + throw new Error('Not found'); + }); + + service.ensureDirectory('/test/dir1/dir2'); + + expect(mockPyodide.FS.mkdir).toHaveBeenCalledWith('/test'); + expect(mockPyodide.FS.mkdir).toHaveBeenCalledWith('/test/dir1'); + expect(mockPyodide.FS.mkdir).toHaveBeenCalledWith('/test/dir1/dir2'); + }); + + it('should handle existing directories', () => { + mockPyodide.FS.stat.mockReturnValue({ mode: 0o040000 }); + mockPyodide.FS.isDir.mockReturnValue(true); + + service.ensureDirectory('/test/dir1/dir2'); + + expect(mockPyodide.FS.mkdir).not.toHaveBeenCalled(); + }); + + it('should replace file with directory if path exists as file', () => { + mockPyodide.FS.stat.mockReturnValue({ mode: 0o100644 }); + mockPyodide.FS.isDir.mockReturnValue(false); + + service.ensureDirectory('/test/dir1'); + + expect(mockPyodide.FS.unlink).toHaveBeenCalledWith('/test/dir1'); + expect(mockPyodide.FS.mkdir).toHaveBeenCalledWith('/test/dir1'); + }); + }); + + describe('removeStaleItems', () => { + it('should remove files and directories not in validPaths', () => { + const validPaths = new Set(['/home/pyodide/keep.txt']); + + // Mock readdir to return different results for different paths + mockPyodide.FS.readdir.mockImplementation((path: string) => { + if (path === '/home/pyodide') { + return ['keep.txt', 'remove.txt', 'dir1']; + } + // Return empty directory for recursive calls + return ['.', '..']; + }); + + const mockStats: MockStats = { + 'keep.txt': { mode: 0o100644 }, + 'remove.txt': { mode: 0o100644 }, + dir1: { mode: 0o040000 }, + }; + + mockPyodide.FS.stat.mockImplementation((path: string) => { + const name = path.split('/').pop() || ''; + const stat = mockStats[name]; + if (!stat) { + // Return a regular file stat for unknown paths to prevent recursion + return { mode: 0o100644 }; + } + return stat; + }); + mockPyodide.FS.isDir.mockImplementation( + (mode: number) => mode === 0o040000, + ); + + service.removeStaleItems('/home/pyodide', validPaths); + + expect(mockPyodide.FS.unlink).toHaveBeenCalledWith( + '/home/pyodide/remove.txt', + ); + expect(mockPyodide.FS.rmdir).toHaveBeenCalledWith('/home/pyodide/dir1'); + expect(mockPyodide.FS.unlink).not.toHaveBeenCalledWith( + '/home/pyodide/keep.txt', + ); + }); + }); + + describe('syncToIndexedDB', () => { + it('should sync to IndexedDB successfully', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error unused-vars + (populate: boolean, callback: FSCallback) => callback(null), + ); + + await service.syncToIndexedDB(); + + expect(mockPyodide.FS.syncfs).toHaveBeenCalledWith( + false, + expect.any(Function), + ); + }); + + it('should handle sync errors', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error unused-vars + (populate: boolean, callback: FSCallback) => + callback(new Error('Sync failed')), + ); + + await expect(service.syncToIndexedDB()).rejects.toThrow('Sync failed'); + }); + }); + + describe('syncFromIndexedDB', () => { + it('should sync from IndexedDB successfully', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error unused-vars + (populate: boolean, callback: FSCallback) => callback(null), + ); + + await service.syncFromIndexedDB(); + + expect(mockPyodide.FS.syncfs).toHaveBeenCalledWith( + true, + expect.any(Function), + ); + }); + + it('should handle sync errors', async () => { + mockPyodide.FS.syncfs.mockImplementation( + // @ts-expect-error unused-vars + (populate: boolean, callback: FSCallback) => + callback(new Error('Sync failed')), + ); + + await expect(service.syncFromIndexedDB()).rejects.toThrow('Sync failed'); + }); + }); +}); diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/job-service.test.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/job-service.test.ts new file mode 100644 index 000000000..c6c12d5a1 --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/__tests__/job-service.test.ts @@ -0,0 +1,249 @@ +import type { AddFileToInboxResponse, AddFileToJobRequest } from '../__mocks__/shinkai-message-ts'; +import type { DirectoryContent } from '../__mocks__/shinkai-message-ts'; +import { FileSystemEntry, IFileSystemService } from '../file-system-service'; +import { JobService } from '../job-service'; + +describe('JobService', () => { + let mockFileSystemService: jest.Mocked; + let mockDownloadFile: jest.Mock; + let mockAddFileToJob: jest.Mock; + let jobService: JobService; + + beforeEach(() => { + mockFileSystemService = { + initialize: jest.fn(), + readContents: jest.fn(), + readContentsWithMtime: jest.fn(), + writeFile: jest.fn(), + readFile: jest.fn(), + ensureDirectory: jest.fn(), + removeStaleItems: jest.fn(), + syncToIndexedDB: jest.fn(), + syncFromIndexedDB: jest.fn(), + }; + + mockDownloadFile = jest.fn(); + mockAddFileToJob = jest.fn(); + + jobService = new JobService({ + fileSystemService: mockFileSystemService, + downloadFile: mockDownloadFile, + addFileToJob: mockAddFileToJob, + nodeAddress: 'test-node', + token: 'test-token', + jobId: 'test-job', + }); + }); + + describe('syncJobFilesToIDBFS', () => { + const mockContents: DirectoryContent[] = [ + { + name: 'file1.txt', + path: 'file1.txt', + is_directory: false, + children: null, + created_time: '2024-01-01T00:00:00Z', + modified_time: '2024-01-01T00:00:00Z', + has_embeddings: false, + size: 100, + }, + { + name: 'dir1', + path: 'dir1', + is_directory: true, + children: [ + { + name: 'file2.txt', + path: 'dir1/file2.txt', + is_directory: false, + children: null, + created_time: '2024-01-01T00:00:00Z', + modified_time: '2024-01-01T00:00:00Z', + has_embeddings: false, + size: 100, + }, + ], + created_time: '2024-01-01T00:00:00Z', + modified_time: '2024-01-01T00:00:00Z', + has_embeddings: false, + size: 0, + }, + ]; + + it('should sync files correctly', async () => { + mockFileSystemService.readContentsWithMtime.mockReturnValue([]); + mockDownloadFile.mockResolvedValue('new content'); + + await jobService.syncJobFilesToIDBFS(mockContents); + + expect(mockFileSystemService.removeStaleItems).toHaveBeenCalled(); + expect(mockFileSystemService.ensureDirectory).toHaveBeenCalledWith('/home/pyodide/dir1'); + expect(mockDownloadFile).toHaveBeenCalledWith({ + nodeAddress: 'test-node', + token: 'test-token', + path: 'file1.txt', + }); + expect(mockFileSystemService.writeFile).toHaveBeenCalled(); + expect(mockFileSystemService.syncToIndexedDB).toHaveBeenCalled(); + }); + + it('should skip unchanged files', async () => { + mockFileSystemService.readFile.mockReturnValue('same content'); + mockDownloadFile.mockResolvedValue('same content'); + + await jobService.syncJobFilesToIDBFS([ + { + name: 'unchanged.txt', + path: 'unchanged.txt', + is_directory: false, + children: null, + created_time: '2024-01-01T00:00:00Z', + modified_time: '2024-01-01T00:00:00Z', + has_embeddings: false, + size: 100, + }, + ]); + + expect(mockFileSystemService.writeFile).not.toHaveBeenCalled(); + }); + + it('should handle empty contents', async () => { + await jobService.syncJobFilesToIDBFS(null); + + expect(mockFileSystemService.removeStaleItems).toHaveBeenCalledWith('/home/pyodide', new Set()); + expect(mockFileSystemService.syncToIndexedDB).toHaveBeenCalled(); + }); + + it('should handle errors', async () => { + mockFileSystemService.removeStaleItems.mockImplementation(() => { + throw new Error('Sync failed'); + }); + + await expect(jobService.syncJobFilesToIDBFS(mockContents)).rejects.toThrow('Sync failed'); + }); + }); + + describe('compareAndUploadFiles', () => { + const mockIDBFSContents: FileSystemEntry[] = [ + { + name: 'file1.txt', + type: 'file', + content: 'new content', + mtimeMs: Date.now() + 1000, // Future time to ensure it's newer + }, + { + name: 'dir1', + type: 'directory', + contents: [ + { + name: 'file2.txt', + type: 'file', + content: 'old content', + mtimeMs: Date.now() - 1000, // Past time to ensure it's older + }, + ], + }, + ]; + + it('should upload changed files', async () => { + mockFileSystemService.readContentsWithMtime.mockReturnValue(mockIDBFSContents); + mockAddFileToJob.mockResolvedValue({ + message: 'File uploaded successfully', + filename: 'file1.txt' + } as AddFileToInboxResponse); + + const timeBefore = Date.now(); + await jobService.compareAndUploadFiles(timeBefore); + + expect(mockAddFileToJob).toHaveBeenCalledWith( + 'test-node', + 'test-token', + expect.objectContaining({ + filename: 'file1.txt', + job_id: 'test-job', + } as AddFileToJobRequest) + ); + }); + + it('should skip unchanged files', async () => { + const oldTime = Date.now() - 1000; + mockFileSystemService.readContentsWithMtime.mockReturnValue([ + { + name: 'old.txt', + type: 'file', + content: 'old content', + mtimeMs: oldTime, + }, + ]); + + await jobService.compareAndUploadFiles(Date.now()); + + expect(mockAddFileToJob).not.toHaveBeenCalled(); + }); + + it('should handle empty IDBFS', async () => { + mockFileSystemService.readContentsWithMtime.mockReturnValue(null); + + await jobService.compareAndUploadFiles(Date.now()); + + expect(mockAddFileToJob).not.toHaveBeenCalled(); + }); + + it('should handle upload errors', async () => { + mockFileSystemService.readContentsWithMtime.mockReturnValue(mockIDBFSContents); + mockAddFileToJob.mockRejectedValue(new Error('Upload failed')); + + const timeBefore = Date.now(); + await jobService.compareAndUploadFiles(timeBefore); + + // Should not throw error, just log it + expect(mockAddFileToJob).toHaveBeenCalled(); + }); + }); + + describe('syncFileToIDBFS', () => { + it('should sync file correctly', async () => { + mockDownloadFile.mockResolvedValue('file content'); + + await jobService.syncFileToIDBFS({ + path: 'test.txt', + name: 'test.txt', + }); + + expect(mockDownloadFile).toHaveBeenCalledWith({ + nodeAddress: 'test-node', + token: 'test-token', + path: 'test.txt', + }); + expect(mockFileSystemService.writeFile).toHaveBeenCalledWith( + '/home/pyodide/test.txt', + 'file content' + ); + }); + + it('should handle download errors', async () => { + mockDownloadFile.mockRejectedValue(new Error('Download failed')); + + await expect( + jobService.syncFileToIDBFS({ + path: 'test.txt', + name: 'test.txt', + }) + ).rejects.toThrow('Download failed'); + }); + + it('should handle write errors', async () => { + mockDownloadFile.mockResolvedValue('file content'); + mockFileSystemService.writeFile.mockImplementation(() => { + throw new Error('Write failed'); + }); + + await expect( + jobService.syncFileToIDBFS({ + path: 'test.txt', + name: 'test.txt', + }) + ).rejects.toThrow('Write failed'); + }); + }); +}); \ No newline at end of file diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/services/file-system-service.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/file-system-service.ts new file mode 100644 index 000000000..efc7cc346 --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/file-system-service.ts @@ -0,0 +1,213 @@ +import { PyodideInterface } from 'pyodide'; + +export type FileSystemEntry = { + name: string; + type: 'directory' | 'file'; + content?: string; + contents?: FileSystemEntry[]; + mtimeMs?: number; +}; + +export interface IFileSystemService { + initialize(): Promise; + readContents(path: string): FileSystemEntry[] | null; + readContentsWithMtime(path: string): FileSystemEntry[] | null; + writeFile(path: string, content: string): void; + readFile(path: string): string | undefined; + ensureDirectory(dirPath: string): void; + removeStaleItems(dirPath: string, validPaths: Set): void; + syncToIndexedDB(): Promise; + syncFromIndexedDB(): Promise; +} + +export class PyodideFileSystemService implements IFileSystemService { + private pyodide: PyodideInterface; + private rootPath: string; + + constructor(pyodide: PyodideInterface, rootPath = '/home/pyodide') { + this.pyodide = pyodide; + this.rootPath = rootPath; + } + + async initialize(): Promise { + try { + this.pyodide.FS.mount( + this.pyodide.FS.filesystems.IDBFS, + {}, + this.rootPath + ); + await this.syncFromIndexedDB(); + } catch (error) { + console.error('Failed to initialize file system:', error); + throw error; + } + } + + readContents(path: string): FileSystemEntry[] | null { + try { + const entries = this.pyodide.FS.readdir(path); + const contents = entries.filter((entry: string) => + entry !== '.' && entry !== '..' && entry !== '.matplotlib' + ); + + return contents.map((entry: string) => { + const fullPath = `${path}/${entry}`; + const stat = this.pyodide.FS.stat(fullPath); + const isDirectory = this.pyodide.FS.isDir(stat.mode); + + if (isDirectory) { + return { + name: entry, + type: 'directory' as const, + contents: this.readContents(fullPath) + }; + } else { + const content = this.pyodide.FS.readFile(fullPath, { encoding: 'utf8' }); + return { + name: entry, + type: 'file' as const, + content: content as string + }; + } + }); + } catch (error) { + console.error(`Error reading ${path}:`, error); + return null; + } + } + + readContentsWithMtime(path: string): FileSystemEntry[] | null { + try { + const entries = this.pyodide.FS.readdir(path); + const contents = entries.filter((entry: string) => + entry !== '.' && entry !== '..' && entry !== '.matplotlib' + ); + + return contents.map((entry: string) => { + const fullPath = `${path}/${entry}`; + const stat = this.pyodide.FS.stat(fullPath); + const isDirectory = this.pyodide.FS.isDir(stat.mode); + const mtimeMs = stat ? stat.mtime * 1000 : 0; + + if (isDirectory) { + return { + name: entry, + type: 'directory' as const, + contents: this.readContentsWithMtime(fullPath) + }; + } else { + const content = this.pyodide.FS.readFile(fullPath, { encoding: 'utf8' }); + return { + name: entry, + type: 'file' as const, + content: content as string, + mtimeMs + }; + } + }); + } catch (error) { + console.error(`Error reading ${path}:`, error); + return null; + } + } + + writeFile(path: string, content: string): void { + try { + this.pyodide.FS.writeFile(path, content, { encoding: 'utf8' }); + } catch (error) { + console.error(`Failed to write file ${path}:`, error); + throw error; + } + } + + readFile(path: string): string | undefined { + try { + return this.pyodide.FS.readFile(path, { encoding: 'utf8' }); + } catch (error) { + console.error(`Failed to read file ${path}:`, error); + return undefined; + } + } + + ensureDirectory(dirPath: string): void { + if (dirPath === this.rootPath) { + return; + } + + const parts = dirPath.split('/').filter(Boolean); + let currentPath = ''; + + for (const part of parts) { + currentPath += '/' + part; + try { + const stat = this.pyodide.FS.stat(currentPath); + if (!this.pyodide.FS.isDir(stat.mode)) { + this.pyodide.FS.unlink(currentPath); + this.pyodide.FS.mkdir(currentPath); + } + } catch (err) { + try { + this.pyodide.FS.mkdir(currentPath); + } catch (mkdirErr) { + console.error(`Failed to create directory ${currentPath}:`, mkdirErr); + throw mkdirErr; + } + } + } + } + + removeStaleItems(dirPath: string, validPaths: Set): void { + const entries = this.pyodide.FS.readdir(dirPath); + for (const entry of entries) { + if (entry === '.' || entry === '..') continue; + const fullPath = `${dirPath}/${entry}`; + const stat = this.pyodide.FS.stat(fullPath); + if (this.pyodide.FS.isDir(stat.mode)) { + this.removeStaleItems(fullPath, validPaths); + if (!validPaths.has(fullPath)) { + try { + this.pyodide.FS.rmdir(fullPath); + } catch (err) { + console.error(`Failed removing directory ${fullPath}:`, err); + } + } + } else { + if (!validPaths.has(fullPath)) { + try { + this.pyodide.FS.unlink(fullPath); + } catch (err) { + console.error(`Failed removing file ${fullPath}:`, err); + } + } + } + } + } + + async syncToIndexedDB(): Promise { + return new Promise((resolve, reject) => { + this.pyodide.FS.syncfs(false, (err: Error | null) => { + if (err) { + console.error('Failed to sync to IndexedDB:', err); + reject(err); + } else { + console.log('Successfully synced to IndexedDB'); + resolve(); + } + }); + }); + } + + async syncFromIndexedDB(): Promise { + return new Promise((resolve, reject) => { + this.pyodide.FS.syncfs(true, (err: Error | null) => { + if (err) { + console.error('Failed to sync from IndexedDB:', err); + reject(err); + } else { + console.log('Successfully synced from IndexedDB'); + resolve(); + } + }); + }); + } +} \ No newline at end of file diff --git a/apps/shinkai-desktop/src/components/chat/python-code-runner/services/job-service.ts b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/job-service.ts new file mode 100644 index 000000000..0f4da357c --- /dev/null +++ b/apps/shinkai-desktop/src/components/chat/python-code-runner/services/job-service.ts @@ -0,0 +1,175 @@ +import type { DirectoryContent } from '@shinkai_network/shinkai-message-ts/api/vector-fs/types'; + +import type { AddFileToInboxResponse, AddFileToJobRequest } from './__mocks__/shinkai-message-ts'; +import { FileSystemEntry, IFileSystemService } from './file-system-service'; + +export interface IJobService { + syncJobFilesToIDBFS(contents: DirectoryContent[] | null): Promise; + compareAndUploadFiles(timeBefore: number): Promise; + syncFileToIDBFS(item: { path: string; name: string }): Promise; +} + +export interface JobServiceDependencies { + fileSystemService: IFileSystemService; + downloadFile: (params: { nodeAddress: string; token: string; path: string }) => Promise; + addFileToJob: (nodeAddress: string, bearerToken: string, payload: AddFileToJobRequest) => Promise; + nodeAddress: string; + token: string; + jobId: string; +} + +export class JobService implements IJobService { + private fileSystemService: IFileSystemService; + private downloadFile: (params: { nodeAddress: string; token: string; path: string }) => Promise; + private addFileToJob: (nodeAddress: string, bearerToken: string, payload: AddFileToJobRequest) => Promise; + private nodeAddress: string; + private token: string; + private jobId: string; + + constructor(dependencies: JobServiceDependencies) { + this.fileSystemService = dependencies.fileSystemService; + this.downloadFile = dependencies.downloadFile; + this.addFileToJob = dependencies.addFileToJob; + this.nodeAddress = dependencies.nodeAddress; + this.token = dependencies.token; + this.jobId = dependencies.jobId; + } + + async syncJobFilesToIDBFS(contents: DirectoryContent[] | null): Promise { + console.time('Sync Job Files to IDBFS'); + try { + // Log current state + console.group('Initial State'); + console.log('Job Contents:', contents ? JSON.stringify(contents, null, 2) : 'empty'); + const currentIDBFSContents = this.fileSystemService.readContentsWithMtime('/home/pyodide'); + console.log('Current IDBFS Contents:', JSON.stringify(currentIDBFSContents, null, 2)); + console.groupEnd(); + + // Create empty Set if no job contents + const jobPathsSet = new Set(); + if (contents) { + for (const entry of contents) { + const fullDirPath = `/home/pyodide/${entry.name}`; + jobPathsSet.add(fullDirPath); + if (!entry.is_directory) { + jobPathsSet.add(fullDirPath); + } + } + } + + // Remove everything not in jobPathsSet + console.group('Removing Stale Items'); + this.fileSystemService.removeStaleItems('/home/pyodide', jobPathsSet); + console.groupEnd(); + + // Only proceed with syncing if we have job contents + if (contents && contents.length > 0) { + console.group('Syncing New/Updated Items'); + for (const entry of contents) { + const fullPath = `/home/pyodide/${entry.name}`; + + if (entry.is_directory) { + this.fileSystemService.ensureDirectory(fullPath); + } else { + const dirOnly = fullPath.substring(0, fullPath.lastIndexOf('/')); + this.fileSystemService.ensureDirectory(dirOnly); + + const existingContent = this.fileSystemService.readFile(fullPath); + + if (existingContent) { + // Compare with new content before syncing + const currentContent = await this.downloadFile({ + nodeAddress: this.nodeAddress, + token: this.token, + path: entry.path, + }); + + if (existingContent === currentContent) { + console.log(`File ${entry.name} content unchanged, skipping sync`); + continue; + } + } + + await this.syncFileToIDBFS({ + path: entry.path, + name: entry.name + }); + } + } + console.groupEnd(); + } + + // Final sync to IndexedDB + await this.fileSystemService.syncToIndexedDB(); + + // Log final state + console.log('Final IDBFS Contents:', JSON.stringify(this.fileSystemService.readContentsWithMtime('/home/pyodide'), null, 2)); + + } catch (error) { + console.error('Failed to sync job files:', error); + throw error; + } + } + + private async traverseAndUpload( + entries: FileSystemEntry[], + basePath: string, + timeBefore: number + ): Promise { + for (const entry of entries) { + const fullPath = `${basePath}/${entry.name}`; + if (entry.type === 'file' && entry.mtimeMs !== undefined) { + const mtimeInMs = entry.mtimeMs; + if (mtimeInMs > timeBefore) { + console.log(`Uploading changed file: ${fullPath}`); + try { + const blob = new Blob([entry.content ?? ''], { type: 'text/plain' }); + const file = new File([blob], entry.name, { type: 'text/plain' }); + await this.addFileToJob(this.nodeAddress, this.token, { + filename: fullPath.replace('/home/pyodide/', ''), + job_id: this.jobId, + file, + }); + } catch (error) { + console.error(`Failed to upload file ${fullPath}:`, error); + } + } + } else if (entry.type === 'directory' && entry.contents) { + await this.traverseAndUpload(entry.contents, fullPath, timeBefore); + } + } + } + + async compareAndUploadFiles(timeBefore: number): Promise { + console.time('Compare and Upload Files'); + try { + const idbfsContents = this.fileSystemService.readContentsWithMtime('/home/pyodide'); + if (!idbfsContents) { + console.warn('No contents found in /home/pyodide.'); + return; + } + + await this.traverseAndUpload(idbfsContents, '/home/pyodide', timeBefore); + } catch (error) { + console.error('Failed to compare/upload files:', error); + } finally { + console.timeEnd('Compare and Upload Files'); + } + } + + async syncFileToIDBFS(item: { path: string; name: string }): Promise { + try { + const content = await this.downloadFile({ + nodeAddress: this.nodeAddress, + token: this.token, + path: item.path, + }); + + this.fileSystemService.writeFile(`/home/pyodide/${item.name}`, content); + console.log(`Synced file ${item.name} to IDBFS`); + } catch (error) { + console.error(`Failed to sync file ${item.name}:`, error); + throw error; + } + } +} \ No newline at end of file diff --git a/libs/shinkai-ui/src/components/chat/python-code-runner/stderr-render.tsx b/apps/shinkai-desktop/src/components/chat/python-code-runner/stderr-render.tsx similarity index 100% rename from libs/shinkai-ui/src/components/chat/python-code-runner/stderr-render.tsx rename to apps/shinkai-desktop/src/components/chat/python-code-runner/stderr-render.tsx diff --git a/libs/shinkai-ui/src/components/chat/python-code-runner/stdout-render.tsx b/apps/shinkai-desktop/src/components/chat/python-code-runner/stdout-render.tsx similarity index 100% rename from libs/shinkai-ui/src/components/chat/python-code-runner/stdout-render.tsx rename to apps/shinkai-desktop/src/components/chat/python-code-runner/stdout-render.tsx diff --git a/apps/shinkai-desktop/src/components/playground-tool/components/tool-playground.tsx b/apps/shinkai-desktop/src/components/playground-tool/components/tool-playground.tsx index 698eb1352..9184987cd 100644 --- a/apps/shinkai-desktop/src/components/playground-tool/components/tool-playground.tsx +++ b/apps/shinkai-desktop/src/components/playground-tool/components/tool-playground.tsx @@ -27,7 +27,6 @@ import { FormItem, FormLabel, JsonForm, - MessageList, ResizableHandle, ResizablePanel, ResizablePanelGroup, @@ -72,6 +71,7 @@ import { toast } from 'sonner'; import { useAuth } from '../../../store/auth'; import { AIModelSelector } from '../../chat/chat-action-bar/ai-update-selection-action-bar'; +import { MessageList } from '../../chat/components/message-list'; import { ToolErrorFallback } from '../error-boundary'; import { CreateToolCodeFormSchema, diff --git a/apps/shinkai-desktop/src/components/sheet/table-chat.tsx b/apps/shinkai-desktop/src/components/sheet/table-chat.tsx index 213692c20..11373b7a8 100644 --- a/apps/shinkai-desktop/src/components/sheet/table-chat.tsx +++ b/apps/shinkai-desktop/src/components/sheet/table-chat.tsx @@ -14,13 +14,7 @@ import { } from '@shinkai_network/shinkai-node-state/v2/constants'; import { useCreateJob } from '@shinkai_network/shinkai-node-state/v2/mutations/createJob/useCreateJob'; import { useSendMessageToJob } from '@shinkai_network/shinkai-node-state/v2/mutations/sendMessageToJob/useSendMessageToJob'; -import { - Button, - Form, - FormField, - Input, - MessageList, -} from '@shinkai_network/shinkai-ui'; +import { Button, Form, FormField, Input } from '@shinkai_network/shinkai-ui'; import { SendIcon } from '@shinkai_network/shinkai-ui/assets'; import { cn } from '@shinkai_network/shinkai-ui/utils'; import { useQueryClient } from '@tanstack/react-query'; @@ -36,6 +30,7 @@ import { AIModelSelector, AiUpdateSelectionActionBar, } from '../chat/chat-action-bar/ai-update-selection-action-bar'; +import { MessageList } from '../chat/components/message-list'; import { useWebSocketMessage } from '../chat/websocket-message'; import { useSheetProjectStore } from './context/table-context'; diff --git a/apps/shinkai-desktop/src/test-setup.ts b/apps/shinkai-desktop/src/test-setup.ts new file mode 100644 index 000000000..f1ed16623 --- /dev/null +++ b/apps/shinkai-desktop/src/test-setup.ts @@ -0,0 +1,22 @@ +// jest-dom adds custom jest matchers for asserting on DOM nodes. +// allows you to do things like: +// expect(element).toHaveTextContent(/react/i) +// learn more: https://github.com/testing-library/jest-dom +import '@testing-library/jest-dom'; + +// Mock console methods to reduce noise in test output +const originalConsoleError = console.error; +const originalConsoleWarn = console.warn; +const originalConsoleLog = console.log; + +beforeAll(() => { + console.error = jest.fn(); + console.warn = jest.fn(); + console.log = jest.fn(); +}); + +afterAll(() => { + console.error = originalConsoleError; + console.warn = originalConsoleWarn; + console.log = originalConsoleLog; +}); diff --git a/apps/shinkai-desktop/tsconfig.spec.json b/apps/shinkai-desktop/tsconfig.spec.json index 0c872b8e3..3217c7634 100644 --- a/apps/shinkai-desktop/tsconfig.spec.json +++ b/apps/shinkai-desktop/tsconfig.spec.json @@ -2,7 +2,13 @@ "extends": "./tsconfig.json", "compilerOptions": { "outDir": "../../dist/out-tsc", - "types": ["vitest/globals", "vitest/importMeta", "vite/client", "node"] + "types": [ + "vitest/globals", + "vitest/importMeta", + "vite/client", + "jest", + "node" + ] }, "include": [ "vite.config.ts", diff --git a/libs/shinkai-message-ts/src/api/methods.ts b/libs/shinkai-message-ts/src/api/methods.ts index 746f57a14..f66e32b11 100644 --- a/libs/shinkai-message-ts/src/api/methods.ts +++ b/libs/shinkai-message-ts/src/api/methods.ts @@ -17,6 +17,7 @@ import { urlJoin } from '../utils/url-join'; import { ShinkaiMessageBuilderWrapper } from '../wasm/ShinkaiMessageBuilderWrapper'; import { ShinkaiNameWrapper } from '../wasm/ShinkaiNameWrapper'; import { uploadFilesToJob } from './jobs'; +import { DirectoryContent } from './vector-fs/types'; export const fetchPublicKey = (nodeAddress: string) => async (): Promise => { @@ -1299,3 +1300,19 @@ export const removeRowsSheet = async ( const data = response.data; return data; }; + +export const retrieveFilesForJob = async ( + nodeAddress: string, + bearerToken: string, + payload: { job_id: string }, +): Promise => { + const response = await httpClient.get( + urlJoin(nodeAddress, '/v2/retrieve_files_for_job'), + { + params: payload, + headers: { Authorization: `Bearer ${bearerToken}` }, + responseType: 'json', + }, + ); + return response.data; +}; diff --git a/libs/shinkai-message-ts/src/api/vector-fs/types.ts b/libs/shinkai-message-ts/src/api/vector-fs/types.ts index 12e4722ee..178ceaa49 100644 --- a/libs/shinkai-message-ts/src/api/vector-fs/types.ts +++ b/libs/shinkai-message-ts/src/api/vector-fs/types.ts @@ -53,3 +53,9 @@ export type RemoveFsItemRequest = { path: string; }; export type RemoveFsItemResponse = string; + +export interface RetrieveFilesForJobRequest { + job_id: string; +} + +export type RetrieveFilesForJobResponse = DirectoryContent[]; diff --git a/libs/shinkai-node-state/src/v2/constants.ts b/libs/shinkai-node-state/src/v2/constants.ts index 9c96d24dd..99cc90f5d 100644 --- a/libs/shinkai-node-state/src/v2/constants.ts +++ b/libs/shinkai-node-state/src/v2/constants.ts @@ -40,6 +40,7 @@ export enum FunctionKeyV2 { GET_RECURRING_TASK_LOGS = 'GET_RECURRING_TASK_LOGS', GET_SHINKAI_FILE_PROTOCOL = 'GET_SHINKAI_FILE_PROTOCOL', GET_ALL_TOOL_ASSETS = 'GET_ALL_TOOL_ASSETS', + GET_JOB_CONTENTS = 'GET_JOB_CONTENTS', } export const DEFAULT_CHAT_CONFIG = { diff --git a/libs/shinkai-node-state/src/v2/queries/getJobContents/index.ts b/libs/shinkai-node-state/src/v2/queries/getJobContents/index.ts new file mode 100644 index 000000000..c4d851d31 --- /dev/null +++ b/libs/shinkai-node-state/src/v2/queries/getJobContents/index.ts @@ -0,0 +1,15 @@ +import { retrieveFilesForJob as retrieveFilesForJobApi } from '@shinkai_network/shinkai-message-ts/api/methods'; + +import { GetJobContentsInput } from './types'; + +export const getJobContents = async ({ + nodeAddress, + jobId, + token, +}: GetJobContentsInput) => { + const response = await retrieveFilesForJobApi(nodeAddress, token, { + job_id: jobId, + }); + + return response; +}; diff --git a/libs/shinkai-node-state/src/v2/queries/getJobContents/types.ts b/libs/shinkai-node-state/src/v2/queries/getJobContents/types.ts new file mode 100644 index 000000000..3643f3a2f --- /dev/null +++ b/libs/shinkai-node-state/src/v2/queries/getJobContents/types.ts @@ -0,0 +1,25 @@ +import { Token } from '@shinkai_network/shinkai-message-ts/api/general/types'; +import { DirectoryContent } from '@shinkai_network/shinkai-message-ts/api/vector-fs/types'; +import { QueryObserverOptions } from '@tanstack/react-query'; + +import { FunctionKeyV2 } from '../../constants'; + +export type GetJobContentsInput = Token & { + nodeAddress: string; + jobId: string; +}; + +export type UseGetJobContents = [ + FunctionKeyV2.GET_JOB_CONTENTS, + GetJobContentsInput, +]; + +export type GetJobContentsOutput = DirectoryContent[]; + +export type Options = QueryObserverOptions< + GetJobContentsOutput, + Error, + GetJobContentsOutput, + GetJobContentsOutput, + UseGetJobContents +>; diff --git a/libs/shinkai-node-state/src/v2/queries/getJobContents/useGetJobContents.ts b/libs/shinkai-node-state/src/v2/queries/getJobContents/useGetJobContents.ts new file mode 100644 index 000000000..5f8e92564 --- /dev/null +++ b/libs/shinkai-node-state/src/v2/queries/getJobContents/useGetJobContents.ts @@ -0,0 +1,17 @@ +import { useQuery } from '@tanstack/react-query'; + +import { FunctionKeyV2 } from '../../constants'; +import { getJobContents } from './index'; +import { GetJobContentsInput, Options } from './types'; + +export const useGetJobContents = ( + input: GetJobContentsInput, + options?: Omit, +) => { + const response = useQuery({ + queryKey: [FunctionKeyV2.GET_JOB_CONTENTS, input], + queryFn: () => getJobContents(input), + ...options, + }); + return response; +}; diff --git a/libs/shinkai-ui/src/components/chat/index.ts b/libs/shinkai-ui/src/components/chat/index.ts index a9b0501a0..4bd878316 100644 --- a/libs/shinkai-ui/src/components/chat/index.ts +++ b/libs/shinkai-ui/src/components/chat/index.ts @@ -2,9 +2,3 @@ export * from './message'; export * from './message-list'; export * from './files-preview'; export * from './chat-input-area'; -export * from './python-code-runner/error-render'; -export * from './python-code-runner/output-render'; -export * from './python-code-runner/python-code-runner'; -export * from './python-code-runner/python-code-runner-web-worker'; -export * from './python-code-runner/stderr-render'; -export * from './python-code-runner/stdout-render'; diff --git a/libs/shinkai-ui/src/components/chat/message-list.tsx b/libs/shinkai-ui/src/components/chat/message-list.tsx index 6c4570d38..b25c4040b 100644 --- a/libs/shinkai-ui/src/components/chat/message-list.tsx +++ b/libs/shinkai-ui/src/components/chat/message-list.tsx @@ -1,7 +1,4 @@ -import { - Artifact, - ChatConversationInfiniteData, -} from '@shinkai_network/shinkai-node-state/v2/queries/getChatConversation/types'; +import { ChatConversationInfiniteData } from '@shinkai_network/shinkai-node-state/v2/queries/getChatConversation/types'; import { FetchPreviousPageOptions, InfiniteQueryObserverResult, @@ -66,9 +63,6 @@ export const MessageList = ({ regenerateFirstMessage, disabledRetryAndEdit, messageExtra, - setArtifact, - artifacts, - artifact, hidePythonExecution, }: { noMoreMessageLabel: string; @@ -89,9 +83,6 @@ export const MessageList = ({ lastMessageContent?: React.ReactNode; disabledRetryAndEdit?: boolean; messageExtra?: React.ReactNode; - artifacts?: Artifact[]; - setArtifact?: (artifact: Artifact | null) => void; - artifact?: Artifact; hidePythonExecution?: boolean; }) => { const chatContainerRef = useRef(null); @@ -304,8 +295,6 @@ export const MessageList = ({ return ( ); })} diff --git a/libs/shinkai-ui/src/components/chat/message.tsx b/libs/shinkai-ui/src/components/chat/message.tsx index 3ead83982..6b82c4d42 100644 --- a/libs/shinkai-ui/src/components/chat/message.tsx +++ b/libs/shinkai-ui/src/components/chat/message.tsx @@ -40,7 +40,7 @@ import { } from '../tooltip'; import { ChatInputArea } from './chat-input-area'; import { FileList } from './files-preview'; -import { PythonCodeRunner } from './python-code-runner/python-code-runner'; +// import { PythonCodeRunner } from './python-code-runner/python-code-runner'; export const extractErrorPropertyOrContent = ( content: string, @@ -71,8 +71,7 @@ type MessageProps = { disabledEdit?: boolean; handleEditMessage?: (message: string) => void; messageExtra?: React.ReactNode; - setArtifact?: (artifact: Artifact | null) => void; - setArtifactPanel?: (open: boolean) => void; + artifacts?: Artifact[]; artifact?: Artifact; hidePythonExecution?: boolean; @@ -160,9 +159,6 @@ const MessageBase = ({ disabledRetry, disabledEdit, handleEditMessage, - setArtifact, - artifacts, - artifact, hidePythonExecution, }: MessageProps) => { const { t } = useTranslation(); @@ -377,9 +373,9 @@ const MessageBase = ({
)} - {pythonCode && !hidePythonExecution && ( - - )} + {/*{pythonCode && !hidePythonExecution && (*/} + {/* */} + {/*)}*/} {message.role === 'user' && !!message.attachments.length && ( new Promise((resolve) => setTimeout(resolve, ms)); - -type PythonCodeRunnerProps = { - code: string; -}; - -// Define more specific message types -type PageMessage = { - type: 'page'; - method: 'GET' | 'POST'; // New field to specify the method - meta: string; // URL - headers: Record; // Headers - body?: string; // Optional body content for POST - sharedBuffer: SharedArrayBuffer; -}; - -type RunDoneMessage = { - type: 'run-done'; - payload: RunResult; -}; - -type WorkerMessage = PageMessage | RunDoneMessage; - -// Type guard functions -function isPageMessage(message: WorkerMessage): message is PageMessage { - return message.type === 'page'; -} - -function isRunDoneMessage(message: WorkerMessage): message is RunDoneMessage { - return message.type === 'run-done'; -} - -export const usePythonRunnerRunMutation = ( - options?: UseMutationOptions, -) => { - const response = useMutation({ - mutationFn: async (params: { code: string }): Promise => { - const worker = new PythonRunnerWorker(); - - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(new Error('execution timed out')); - }, 120000); // 2 minutes - - worker.onmessage = async (event: { data: WorkerMessage }) => { - if (isPageMessage(event.data)) { - const { method, meta: url, headers, body, sharedBuffer } = event.data; - console.log(`main thread> ${method.toLowerCase()}ing page`, url); - console.log('main thread> headers: ', headers); - - const syncArray = new Int32Array(sharedBuffer, 0, 1); - const dataArray = new Uint8Array(sharedBuffer, 4); - - const bufferSize = 512 * 1024; // Start with 512Kb - const maxBufferSize = 100 * 1024 * 1024; // Set a maximum buffer size, e.g., 100MB - let success = false; - - while (bufferSize <= maxBufferSize && !success) { - try { - console.log(`main thread> ${method.toLowerCase()}ing page`, url); - const response = await invoke<{ - status: number; - headers: Record; - body: string; - }>(method === 'GET' ? 'get_request' : 'post_request', { - url, - customHeaders: JSON.stringify(headers), - ...(method === 'POST' && { body: JSON.stringify(body) }), // Ensure body is a string - }); - console.log(`main thread> ${method.toLowerCase()} response`, response); - - if (response.status >= 200 && response.status < 300) { - const textEncoder = new TextEncoder(); - const encodedData = textEncoder.encode(response.body); - - console.log('Required buffer size:', encodedData.length); // Log the required buffer size - - let offset = 0; - while (offset < encodedData.length) { - const chunkSize = Math.min(dataArray.length, encodedData.length - offset); - dataArray.set(encodedData.subarray(offset, offset + chunkSize)); - offset += chunkSize; - - // Indicate that a chunk is ready - syncArray[0] = 2; // New number to indicate chunk ready - console.log('main thread> Notifying Atomics with chunk ready'); - Atomics.notify(syncArray, 0); - - // Polling loop to wait for the other end to be ready for the next chunk - while (syncArray[0] === 2) { - await delay(25); // Wait for 25ms before checking again - } - } - - // Indicate success after all chunks are sent - syncArray[0] = 1; // Indicate success - console.log('main thread> Notifying Atomics with success'); - Atomics.notify(syncArray, 0); - success = true; - } else { - throw new Error(`HTTP Error: ${response.status}`); - } - } catch (error) { - let errorMessage = 'Unknown error'; - if (error instanceof Error) { - errorMessage = error.message; - } - console.error(`main thread> error using ${method.toLowerCase()} with page`, errorMessage); - - const textEncoder = new TextEncoder(); - const encodedError = textEncoder.encode(errorMessage); - - if (encodedError.length <= dataArray.length) { - dataArray.set(encodedError); - } else { - console.warn('Error message too long to fit in buffer'); - } - - console.log('main thread> Notifying Atomics with error'); - syncArray[0] = -1; // Indicate error - Atomics.notify(syncArray, 0); - - // Await the delay before rejecting - await delay(10); - reject(new Error(`Failed to ${method.toLowerCase()} page: ` + errorMessage)); - return; // Exit the loop and function after rejection - } - } - } else if (isRunDoneMessage(event.data)) { - clearTimeout(timeout); - console.log('main thread> worker event', event); - resolve(event.data.payload); - } - }; - - worker.onerror = (error: { message: string }) => { - console.log('worker error', error); - clearTimeout(timeout); - reject(new Error(`worker error: ${error.message}`)); - }; - - worker.postMessage({ type: 'run', payload: { code: params.code } }); - }).finally(() => { - worker.terminate(); - }); - }, - ...options, - onSuccess: (...onSuccessParameters) => { - if (options?.onSuccess) { - options.onSuccess(...onSuccessParameters); - } - }, - }); - return { ...response }; -}; - -export const PythonCodeRunner = ({ code }: PythonCodeRunnerProps) => { - const i18n = useTranslation(); - const { - mutateAsync: run, - data: runResult, - isPending, - } = usePythonRunnerRunMutation(); - return ( -
- - - {!isPending && runResult && ( - - - - )} - -
- ); -};