Skip to content

Commit

Permalink
Pyodide IDBFS Sync (#588)
Browse files Browse the repository at this point in the history
* playing around

* update

* fix recursiveness

* add tests and refactor

* fix tests

* unify message component, rearrange files and setup jest

* remove jest from shinkai ui

---------

Co-authored-by: paulclindo <[email protected]>
  • Loading branch information
nicarq and paulclindo authored Jan 7, 2025
1 parent ac28f16 commit b4b2270
Show file tree
Hide file tree
Showing 30 changed files with 1,618 additions and 313 deletions.
11 changes: 11 additions & 0 deletions apps/shinkai-desktop/jest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
export default {
displayName: 'shinkai-desktop',
preset: '../../jest.preset.js',
transform: {
'^.+\\.[tj]sx?$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
},
moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx'],
coverageDirectory: '../../coverage/apps/shinkai-desktop',
setupFilesAfterEnv: ['<rootDir>/src/test-setup.ts'],
testEnvironment: 'jsdom',
};
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ export const MessageList = ({
regenerateFirstMessage,
disabledRetryAndEdit,
messageExtra,
hidePythonExecution,
}: {
noMoreMessageLabel: string;
isSuccess: boolean;
Expand All @@ -85,6 +86,7 @@ export const MessageList = ({
lastMessageContent?: React.ReactNode;
disabledRetryAndEdit?: boolean;
messageExtra?: React.ReactNode;
hidePythonExecution?: boolean;
}) => {
const chatContainerRef = useRef<HTMLDivElement>(null);
const previousChatHeightRef = useRef<number>(0);
Expand Down Expand Up @@ -302,6 +304,7 @@ export const MessageList = ({
? handleFirstMessageRetry
: handleRetryMessage
}
hidePythonExecution={hidePythonExecution}
key={`${message.messageId}::${messageIndex}`}
message={message}
messageId={message.messageId}
Expand Down
19 changes: 17 additions & 2 deletions apps/shinkai-desktop/src/components/chat/components/message.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
ToolArgs,
ToolStatusType,
} from '@shinkai_network/shinkai-message-ts/api/general/types';
import { extractJobIdFromInbox } from '@shinkai_network/shinkai-message-ts/utils';
import { FormattedMessage } from '@shinkai_network/shinkai-node-state/v2/queries/getChatConversation/types';
import {
Accordion,
Expand All @@ -22,7 +23,6 @@ import {
Form,
FormField,
MarkdownText,
PythonCodeRunner,
Tooltip,
TooltipContent,
TooltipPortal,
Expand Down Expand Up @@ -50,9 +50,11 @@ import { useForm } from 'react-hook-form';
import { Link } from 'react-router-dom';
import { z } from 'zod';

import { useAuth } from '../../../store/auth';
import { useOAuth } from '../../../store/oauth';
import { oauthUrlMatcherFromErrorMessage } from '../../../utils/oauth';
import { useChatStore } from '../context/chat-context';
import { PythonCodeRunner } from '../python-code-runner/python-code-runner';

export const extractErrorPropertyOrContent = (
content: string,
Expand Down Expand Up @@ -83,6 +85,7 @@ type MessageProps = {
disabledEdit?: boolean;
handleEditMessage?: (message: string) => void;
messageExtra?: React.ReactNode;
hidePythonExecution?: boolean;
};

const actionBar = {
Expand Down Expand Up @@ -162,6 +165,7 @@ type EditMessageFormSchema = z.infer<typeof editMessageFormSchema>;
export const MessageBase = ({
message,
// messageId,
hidePythonExecution,
isPending,
handleRetryMessage,
disabledRetry,
Expand Down Expand Up @@ -211,6 +215,8 @@ export const MessageBase = ({

const { setOauthModalVisible } = useOAuth();

const auth = useAuth((state) => state.auth);

return (
<motion.div
animate="rest"
Expand Down Expand Up @@ -411,7 +417,16 @@ export const MessageBase = ({
<DotsLoader />
</div>
)}
{pythonCode && <PythonCodeRunner code={pythonCode} />}
{pythonCode && !hidePythonExecution && (
<PythonCodeRunner
code={pythonCode}
jobId={extractJobIdFromInbox(
message.metadata.inboxId ?? '',
)}
nodeAddress={auth?.node_address ?? ''}
token={auth?.api_v2_key ?? ''}
/>
)}

{oauthUrl && (
<div className="mt-4 flex flex-col items-start rounded-lg bg-gray-900 p-4 shadow-md">
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { renderHook } from '@testing-library/react';
import { loadPyodide, PyodideInterface } from 'pyodide';

import { usePyodideInstance } from '../usePyodideInstance';

// Mock pyodide
jest.mock('pyodide', () => ({
loadPyodide: jest.fn(),
}));

describe('usePyodideInstance', () => {
let mockPyodide: jest.Mocked<PyodideInterface>;

beforeEach(() => {
// Reset mocks
jest.clearAllMocks();

// Create mock Pyodide instance
mockPyodide = {
FS: {
mount: jest.fn(),
readdir: jest.fn(),
stat: jest.fn(),
isDir: jest.fn(),
readFile: jest.fn(),
writeFile: jest.fn(),
unlink: jest.fn(),
mkdir: jest.fn(),
rmdir: jest.fn(),
syncfs: jest.fn(),
filesystems: {
IDBFS: 'IDBFS',
},
},
} as unknown as jest.Mocked<PyodideInterface>;

// Mock loadPyodide to return our mock instance
(loadPyodide as jest.Mock).mockResolvedValue(mockPyodide);
});

it('should initialize Pyodide and file system service', async () => {
mockPyodide.FS.syncfs.mockImplementation(
// @ts-expect-error populate
(populate: boolean, callback: (err: Error | null) => void) =>
callback(null),
);

const { result } = renderHook(() => usePyodideInstance());

// Initially, both pyodide and fileSystemService should be null
expect(result.current.pyodide).toBeNull();
expect(result.current.fileSystemService).toBeNull();

// Initialize
const { pyodide, fileSystemService } =
await result.current.initializePyodide();

// After initialization
expect(pyodide).toBe(mockPyodide);
expect(fileSystemService).toBeDefined();
expect(loadPyodide).toHaveBeenCalledWith({
indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.26.2/full/',
stdout: console.log,
stderr: console.error,
});
expect(mockPyodide.FS.mount).toHaveBeenCalledWith(
'IDBFS',
{},
'/home/pyodide',
);
expect(mockPyodide.FS.syncfs).toHaveBeenCalledWith(
true,
expect.any(Function),
);
});

it('should reuse existing Pyodide instance', async () => {
mockPyodide.FS.syncfs.mockImplementation(
// @ts-expect-error populate
(populate: boolean, callback: (err: Error | null) => void) =>
callback(null),
);

const { result } = renderHook(() => usePyodideInstance());

// First initialization
const first = await result.current.initializePyodide();
expect(loadPyodide).toHaveBeenCalledTimes(1);

// Second initialization
const second = await result.current.initializePyodide();
expect(loadPyodide).toHaveBeenCalledTimes(1); // Should not be called again
expect(second.pyodide).toBe(first.pyodide);
expect(second.fileSystemService).toBe(first.fileSystemService);
});

it('should handle initialization errors', async () => {
(loadPyodide as jest.Mock).mockRejectedValue(
new Error('Failed to load Pyodide'),
);

const { result } = renderHook(() => usePyodideInstance());

await expect(result.current.initializePyodide()).rejects.toThrow(
'Failed to load Pyodide',
);
});

it('should handle file system initialization errors', async () => {
mockPyodide.FS.syncfs.mockImplementation(
// @ts-expect-error populate
(populate: boolean, callback: (err: Error | null) => void) =>
callback(new Error('Sync failed')),
);

const { result } = renderHook(() => usePyodideInstance());

await expect(result.current.initializePyodide()).rejects.toThrow(
'Sync failed',
);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import { loadPyodide, PyodideInterface } from 'pyodide';
import { useCallback, useRef } from 'react';

import { IFileSystemService, PyodideFileSystemService } from '../services/file-system-service';

export function usePyodideInstance() {
const pyodideRef = useRef<PyodideInterface | null>(null);
const fileSystemServiceRef = useRef<IFileSystemService | null>(null);

const initializePyodide = useCallback(async () => {
if (pyodideRef.current) {
console.log('Pyodide is already initialized.');
return { pyodide: pyodideRef.current, fileSystemService: fileSystemServiceRef.current! };
}

console.time('initialize pyodide');
const pyodide = await loadPyodide({
indexURL: 'https://cdn.jsdelivr.net/pyodide/v0.26.2/full/',
stdout: console.log,
stderr: console.error,
});
console.log('Pyodide initialized');

pyodideRef.current = pyodide;
fileSystemServiceRef.current = new PyodideFileSystemService(pyodide);

try {
await fileSystemServiceRef.current.initialize();
} catch (error) {
console.error('Failed to initialize file system:', error);
throw error;
}

console.timeEnd('initialize pyodide');
return { pyodide, fileSystemService: fileSystemServiceRef.current };
}, []);

return {
pyodide: pyodideRef.current,
fileSystemService: fileSystemServiceRef.current,
initializePyodide,
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,12 @@ const fetchPage = (
method: 'GET' | 'POST',
body: any = null,
): string => {
console.log('fetchPage called with url:', url);
console.log('fetchPage called with headers:', headers);
console.log('fetchPage called with method:', method);
if (body) {
console.log('fetchPage called with body:', body);
}
// console.log('fetchPage called with url:', url);
// console.log('fetchPage called with headers:', headers);
// console.log('fetchPage called with method:', method);
// if (body) {
// console.log('fetchPage called with body:', body);
// }

const filteredHeaders: Record<string, string> = {};
for (const [key, value] of Object.entries(headers.toJs())) {
Expand Down Expand Up @@ -285,8 +285,6 @@ const fetchPage = (
sharedBuffer,
});

console.log('Posted message to main thread, waiting for response...');

const textDecoder = new TextDecoder();
let result = '';
let moreChunks = true;
Expand All @@ -297,8 +295,6 @@ const fetchPage = (
// This loop will block the thread until syncArray[0] changes
}

console.log('Polling done with status: ', syncArray[0]);

if (syncArray[0] === -1) {
const errorMessage = textDecoder.decode(dataArray);
console.error('Error fetching page:', errorMessage);
Expand All @@ -309,8 +305,6 @@ const fetchPage = (
const chunk = textDecoder.decode(dataArray).replace(/\0/g, '').trim();
result += chunk;

console.log(`Received chunk of length: ${chunk.length}`);

// Check if more chunks are needed
if (syncArray[0] === 1) {
moreChunks = false; // Success, all chunks received
Expand Down Expand Up @@ -377,48 +371,10 @@ const initialize = async () => {
console.timeEnd('initialize');
};

// Function to print contents of a directory
function printDirectoryContents(dirPath: string) {
try {
const entries = pyodide.FS.readdir(dirPath);
const folders: Array<string> = [];
const files: Array<string> = [];

entries.forEach((entry: string) => {
if (entry === '.' || entry === '..') return;
const path = `${dirPath}/${entry}`;
const stat = pyodide.FS.stat(path);
if (pyodide.FS.isDir(stat.mode)) {
folders.push(entry);
} else if (pyodide.FS.isFile(stat.mode)) {
files.push(entry);
}
});

console.log(`Contents of ${dirPath}:`);
console.log('Folders:', folders);
console.log('Files:', files);
} catch (error) {
console.error(`Error reading ${dirPath} directory:`, error);
}
}

// Function to synchronize the filesystem to IndexedDB
const syncFilesystem = async (save = false) => {
return new Promise<void>((resolve, reject) => {
pyodide.FS.syncfs(save, (err: any) => {
printDirectoryContents('/home/pyodide');

printDirectoryContents('/home/web_user');

// Print contents inside the /home directory
printDirectoryContents('/home');

printDirectoryContents('/new_mnt');

// Print contents inside the root directory
printDirectoryContents('/');

if (err) {
console.error('syncfs error:', err);
reject(err);
Expand Down
Loading

0 comments on commit b4b2270

Please sign in to comment.