From b6f9232d3e4ce86dd87d4e57e4da319c5cc3b307 Mon Sep 17 00:00:00 2001 From: Sammy Franklin Date: Thu, 1 Aug 2024 15:13:40 -0700 Subject: [PATCH 01/21] convert main file --- web/README.md | 8 -------- web/src/{main.jsx => main.tsx} | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 web/README.md rename web/src/{main.jsx => main.tsx} (72%) diff --git a/web/README.md b/web/README.md deleted file mode 100644 index f768e33..0000000 --- a/web/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# React + Vite - -This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. - -Currently, two official plugins are available: - -- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh -- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh diff --git a/web/src/main.jsx b/web/src/main.tsx similarity index 72% rename from web/src/main.jsx rename to web/src/main.tsx index 3bb5903..9aa52ff 100644 --- a/web/src/main.jsx +++ b/web/src/main.tsx @@ -3,8 +3,8 @@ import ReactDOM from 'react-dom/client'; import App from './App'; import './index.css'; -ReactDOM.createRoot(document.getElementById('root')).render( +ReactDOM.createRoot(document.getElementById('root')!).render( , -) +); From 930d872626727f84a06a6c68573275526d52b569 Mon Sep 17 00:00:00 2001 From: Sammy Franklin Date: Thu, 1 Aug 2024 15:24:41 -0700 Subject: [PATCH 02/21] convert AddNode file --- .../components/{AddNode.jsx => AddNode.tsx} | 45 +++++++++++++------ web/src/components/Flow.jsx | 2 +- 2 files changed, 33 insertions(+), 14 deletions(-) rename web/src/components/{AddNode.jsx => AddNode.tsx} (58%) diff --git a/web/src/components/AddNode.jsx b/web/src/components/AddNode.tsx similarity index 58% rename from web/src/components/AddNode.jsx rename to web/src/components/AddNode.tsx index 8cda566..df5ad77 100644 --- a/web/src/components/AddNode.jsx +++ b/web/src/components/AddNode.tsx @@ -1,22 +1,27 @@ import React, { useCallback, useRef, useEffect, useState } from 'react'; -import { Button } from 'antd'; +import { Input, Button } from 'antd'; import { PlusOutlined } from '@ant-design/icons'; import { API } from '../api'; +import type { InputRef } from 'antd'; import './add-node.css'; export default function AddNode({position, setNodeTo, addNode}) { - const searchRef = useRef(null); + const searchRef = useRef(null); const [searchText, setSearchText] = useState(''); useEffect(() => { - searchRef.current.focus(); + if (searchRef.current) { + searchRef.current.focus(); + } }, [searchRef]); const onInputChange = (e) => { setSearchText(e.target.value); }; + const onResultClick = useCallback(({type, workflowType}) => { + return () => { if (workflowType) { addNode({ type: 'step', position: setNodeTo, data: API.getNodeProperties(workflowType)}); @@ -24,18 +29,18 @@ export default function AddNode({position, setNodeTo, addNode}) { addNode({ type, position: setNodeTo, data: {}}); } } - }); + }, []); return (

Add Node

-
+ {/*
-
- +
*/} +
@@ -43,16 +48,30 @@ export default function AddNode({position, setNodeTo, addNode}) { } function ResultList({onResultClick, searchText}) { - const results = API.getNodeList(); - const resultNames = results.map((result) => result.name); - searchText = searchText.toLowerCase(); - const filteredResultNames = resultNames.filter((result) => result.toLowerCase().includes(searchText)); + const [nodes, setNodes] = useState([]); + const [results, setResults] = useState([]); + + useEffect(() => { + const loadNodes = async () => { + const nodes = await API.getNodes(); + setNodes(nodes); + }; + + loadNodes(); + }, []); + + useEffect(() => { + const resultNames = nodes.map((result) => result.name); + searchText = searchText.toLowerCase(); + const filteredResultNames = resultNames.filter((result) => result.toLowerCase().includes(searchText)); + setResults(filteredResultNames); + }, [searchText, nodes]); return (
{ - filteredResultNames.map((result, i) => { - return
{result}
+ results.map((result, i) => { + return
{result}
}) }
diff --git a/web/src/components/Flow.jsx b/web/src/components/Flow.jsx index 1ae5fb2..2d5f6e3 100644 --- a/web/src/components/Flow.jsx +++ b/web/src/components/Flow.jsx @@ -174,7 +174,7 @@ export default function Flow({ filename }) { return; } if (event.type === 'dblclick' && !isAddNodeActive) { - setIsAddNodeActive(true); + // setIsAddNodeActive(true); setEventMousePos({ x: event.clientX, y: event.clientY }); setNodeToPos(reactFlowInstance.current.screenToFlowPosition({ x: event.clientX, y: event.clientY })); } From 7931427279abc7af70b343b4eb07c62afbf9e596 Mon Sep 17 00:00:00 2001 From: Sammy Franklin Date: Thu, 1 Aug 2024 17:08:06 -0700 Subject: [PATCH 03/21] convert ContextMenu file --- .../{ContextMenu.jsx => ContextMenu.tsx} | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) rename web/src/components/{ContextMenu.jsx => ContextMenu.tsx} (93%) diff --git a/web/src/components/ContextMenu.jsx b/web/src/components/ContextMenu.tsx similarity index 93% rename from web/src/components/ContextMenu.jsx rename to web/src/components/ContextMenu.tsx index d81ac2d..05645ae 100644 --- a/web/src/components/ContextMenu.jsx +++ b/web/src/components/ContextMenu.tsx @@ -1,11 +1,11 @@ +import React, { useCallback, useEffect, useState, useMemo } from 'react'; import { Menu } from 'antd'; import { useReactFlow, useUpdateNodeInternals } from 'reactflow'; -import { useCallback, useEffect, useState, useMemo } from 'react'; -import { keyRecursively, uniqueIdFrom } from '../utils'; -import { Graph } from '../graph'; -import { useRunState } from '../hooks/RunState'; -import { useAPI } from '../hooks/API'; -import { useNotification } from '../hooks/Notification'; +import { keyRecursively, uniqueIdFrom } from '../utils.ts'; +import { Graph } from '../graph.ts'; +import { useRunState } from '../hooks/RunState.ts'; +import { useAPI } from '../hooks/API.ts'; +import { useNotification } from '../hooks/Notification.ts'; import { SerializationErrorMessages } from './Errors.tsx'; import { useFilename } from '../hooks/Filename.ts'; @@ -18,7 +18,7 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { const notification = useNotification(); const filename = useFilename(); - const NODE_OPTIONS = useMemo(() => [ + const NODE_OPTIONS = useMemo(() => !node || !API ? [] : [ { name: 'Run', disabled: () => API && runState !== 'stopped', @@ -132,6 +132,9 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { ], [node, reactFlowInstance, runState, runStateShouldChange, API, filename]); const GROUP_OPTIONS = useMemo(() => { + if (!node || !API) { + return []; + } const addExport = (isInput, exp) => { const currentExports = isInput ? node.data.exports.inputs : node.data.exports.outputs; const id = uniqueIdFrom(currentExports); @@ -171,7 +174,7 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { if (n.parentId === node.id) { return { ...n, - parentId: null, + parentId: undefined, position: { x: n.position.x + node.position.x, y: n.position.y + node.position.y } }; } @@ -232,6 +235,9 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { } const src = nodes.find((n) => n.id === edge.source); const tgt = nodes.find((n) => n.id === edge.target); + if (!src || !tgt) { + continue; + } if ((src.parentId !== node.id && tgt.parentId === node.id) || (src.parentId === node.id && tgt.parentId !== node.id)) { @@ -267,7 +273,7 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { }]; }, [node, reactFlowInstance, notification]); - const EXPORT_OPTIONS = useMemo(() => [ + const EXPORT_OPTIONS = useMemo(() => !node ? [] : [ { name: 'Edit Label', action: () => { @@ -300,9 +306,13 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { if (nodeType === 'step' || nodeType === 'resource') { return NODE_OPTIONS; } + return NODE_OPTIONS; }; const items = useMemo(() => { + if (!node) { + return []; + } const toReturn = getOptions(node.type).map((option) => { return { label: typeof option.name === 'function' ? option.name() : option.name, @@ -314,6 +324,9 @@ export function NodeContextMenu({ nodeId, top, left, ...props }) { }, [node]); const menuItemOnClick = useCallback(({ key }) => { + if (!node) { + return; + } const actionIndex = parseInt(key); const action = getOptions(node.type)[actionIndex].action; action(); @@ -359,21 +372,22 @@ export function PaneContextMenu({ top, left, close }) { useEffect(() => { const setData = async () => { - const nodes = await API.getNodes(); - if (nodes?.steps && nodes?.resources) { - setApiNodes(nodes); + if (API) { + const nodes = await API.getNodes(); + if (nodes?.steps && nodes?.resources) { + setApiNodes(nodes); + } } }; - if (API) { - setData(); - } + + setData(); }, [API]); const items = useMemo(() => { const { steps, resources } = apiNodes; // Convert Dict Tree to List Tree const toListTree = (dictTree) => { - const listTree = []; + const listTree: any[] = []; for (const key in dictTree) { const node = dictTree[key]; const listItem = { @@ -419,7 +433,7 @@ export function PaneContextMenu({ top, left, close }) { const addStep = useCallback((node) => { const position = screenToFlowPosition({ x: left, y: top }); const type = 'step'; - Object.values(node.parameters).forEach((p) => { + Object.values(node.parameters).forEach((p) => { p.value = p.default; }); const newNode = ({ type, position, data: node }); @@ -430,7 +444,7 @@ export function PaneContextMenu({ top, left, close }) { const addResource = useCallback((node) => { const position = screenToFlowPosition({ x: left, y: top }); const type = 'resource'; - Object.values(node.parameters).forEach((p) => { + Object.values(node.parameters).forEach((p) => { p.value = p.default; }); const newNode = ({ type, position, data: node }); @@ -500,4 +514,4 @@ export function PaneContextMenu({ top, left, close }) { return ( ); -} \ No newline at end of file +} From a3f75daa70ceedbd9d9e40c8c29e036d3cd9e9f9 Mon Sep 17 00:00:00 2001 From: Sammy Franklin Date: Sat, 3 Aug 2024 11:02:49 -0700 Subject: [PATCH 04/21] converted flow file --- web/src/components/AddNode.tsx | 28 +++--- web/src/components/{Flow.jsx => Flow.tsx} | 109 +++++++++++++--------- web/src/graph.ts | 2 +- web/src/hooks/Notification.ts | 2 +- 4 files changed, 86 insertions(+), 55 deletions(-) rename web/src/components/{Flow.jsx => Flow.tsx} (86%) diff --git a/web/src/components/AddNode.tsx b/web/src/components/AddNode.tsx index df5ad77..a91f7b1 100644 --- a/web/src/components/AddNode.tsx +++ b/web/src/components/AddNode.tsx @@ -1,13 +1,16 @@ import React, { useCallback, useRef, useEffect, useState } from 'react'; +import { useReactFlow } from 'reactflow'; import { Input, Button } from 'antd'; import { PlusOutlined } from '@ant-design/icons'; import { API } from '../api'; +import { Graph } from '../graph'; import type { InputRef } from 'antd'; import './add-node.css'; -export default function AddNode({position, setNodeTo, addNode}) { +export default function AddNode({ position, setNodeTo }) { const searchRef = useRef(null); const [searchText, setSearchText] = useState(''); + const { setNodes } = useReactFlow(); useEffect(() => { if (searchRef.current) { @@ -18,21 +21,24 @@ export default function AddNode({position, setNodeTo, addNode}) { const onInputChange = (e) => { setSearchText(e.target.value); }; - - const onResultClick = useCallback(({type, workflowType}) => { - + + const onResultClick = useCallback(({ type, workflowType }) => { + const addNode = (node) => { + setNodes((nodes) => Graph.addNode(nodes, node)); + }; + return () => { if (workflowType) { - addNode({ type: 'step', position: setNodeTo, data: API.getNodeProperties(workflowType)}); + addNode({ type: 'step', position: setNodeTo, data: API.getNodeProperties(workflowType) }); } else { - addNode({ type, position: setNodeTo, data: {}}); + addNode({ type, position: setNodeTo, data: {} }); } } }, []); return ( -
+

Add Node

{/*
@@ -40,14 +46,14 @@ export default function AddNode({position, setNodeTo, addNode}) { Code
*/} - - + +
); } -function ResultList({onResultClick, searchText}) { +function ResultList({ onResultClick, searchText }) { const [nodes, setNodes] = useState([]); const [results, setResults] = useState([]); @@ -71,7 +77,7 @@ function ResultList({onResultClick, searchText}) {
{ results.map((result, i) => { - return
{result}
+ return
{result}
}) }
diff --git a/web/src/components/Flow.jsx b/web/src/components/Flow.tsx similarity index 86% rename from web/src/components/Flow.jsx rename to web/src/components/Flow.tsx index 2d5f6e3..c8ef563 100644 --- a/web/src/components/Flow.jsx +++ b/web/src/components/Flow.tsx @@ -10,26 +10,27 @@ import ReactFlow, { } from 'reactflow'; import { Button, Flex, theme } from 'antd'; import { ClearOutlined, CaretRightOutlined, PauseOutlined } from '@ant-design/icons'; -import { Graph } from '../graph'; -import AddNode from './AddNode'; +import { Graph } from '../graph.ts'; +import AddNode from './AddNode.tsx'; import { WorkflowStep } from './Nodes/Node.jsx'; import { Group, groupIfPossible } from './Nodes/Group.tsx'; import { getHandle, filesystemDragEnd } from '../utils.ts'; import { Resource } from './Nodes/Resource.jsx'; import { Export } from './Nodes/Export.tsx'; -import { NodeContextMenu, PaneContextMenu } from './ContextMenu'; +import { NodeContextMenu, PaneContextMenu } from './ContextMenu.tsx'; import { useAPI, useAPIMessage } from '../hooks/API.ts'; -import { useRunState } from '../hooks/RunState'; +import { useRunState } from '../hooks/RunState.ts'; import { GraphStore } from '../graphstore.ts'; import { NodeConfig } from './NodeConfig.tsx'; import { Subflow } from './Nodes/Subflow.tsx'; import { Monitor } from './Monitor.tsx'; -import { useNotificationInitializer, useNotification } from '../hooks/Notification'; +import { useNotificationInitializer, useNotification } from '../hooks/Notification.ts'; import { SerializationErrorMessages } from './Errors.tsx'; import { useFilename } from '../hooks/Filename.ts'; +import { ReactFlowInstance, Node, Edge, BackgroundVariant } from 'reactflow'; const { useToken } = theme; const makeDroppable = (e) => e.preventDefault(); -const onLoadGraph = async (filename, API) => { +const onLoadGraph = async (filename, API): Promise<[Node[], Edge[]]> => { const file = await API.getFile(filename); if (file?.content) { const graph = JSON.parse(file.content); @@ -40,44 +41,51 @@ const onLoadGraph = async (filename, API) => { return [[], []]; }; +type NodeMenu = { + nodeId: string; + top: number; + left: number; +}; + +type PaneMenu = { + top: number; + left: number; +}; + export default function Flow({ filename }) { const { token } = useToken(); const API = useAPI(); const [nodes, setNodes, onNodesChange] = useNodesState([]); const [edges, setEdges, onEdgesChange] = useEdgesState([]); - const [nodeMenu, setNodeMenu] = useState(null); - const [paneMenu, setPaneMenu] = useState(null); + const [nodeMenu, setNodeMenu] = useState(null); + const [paneMenu, setPaneMenu] = useState(null); const [runState, _] = useRunState(); - const graphStore = useRef(null); + const graphStore = useRef(null); const [notificationCtrl, notificationCtxt] = useNotificationInitializer(); // Coalesce const [isAddNodeActive, setIsAddNodeActive] = useState(false); const [eventMousePos, setEventMousePos] = useState({ x: 0, y: 0 }); const [nodeToPos, setNodeToPos] = useState({ x: 0, y: 0 }); - const reactFlowInstance = useRef(null); - const reactFlowRef = useRef(null); - - - useEffect(() => { - graphStore.current = null; - }, [filename]); + const reactFlowInstance = useRef(null); useEffect(() => { const loadGraph = async () => { - const [nodes, edges] = await onLoadGraph(filename, API); - setNodes(nodes); - setEdges(edges); - graphStore.current = new GraphStore(filename, API, nodes, edges); + if (API) { + /* Setting to empty so that Reactflow's internal edge rendering system is refreshed */ + setNodes([]); + setEdges([]); + + const [nodes, edges] = await onLoadGraph(filename, API); + setNodes(nodes); + setEdges(edges); + graphStore.current = new GraphStore(filename, API!, nodes, edges); + } }; + graphStore.current = null; + loadGraph(); - if (API) { - /* Setting to empty so that Reactflow's internal edge rendering system is refreshed */ - setNodes([]); - setEdges([]); - loadGraph(); - } }, [API, filename]); const nodeTypes = useMemo(() => ({ @@ -88,9 +96,9 @@ export default function Flow({ filename }) { subflow: Subflow, }), []); - const onInitReactFlow = (instance) => { + const onInitReactFlow = useCallback((instance) => { reactFlowInstance.current = instance; - }; + }, [reactFlowInstance]); const onNodesChangeCallback = useCallback((changes) => { setIsAddNodeActive(false); @@ -107,7 +115,7 @@ export default function Flow({ filename }) { return onNodesChange(newChanges); } onNodesChange(changes); - }); + }, []); const onEdgesChangeCallback = useCallback((changes) => { setIsAddNodeActive(false); @@ -124,11 +132,14 @@ export default function Flow({ filename }) { return onEdgesChange(newChanges); } onEdgesChange(changes); - }); + }, []); const onConnect = useCallback((params) => { const targetNode = nodes.find(n => n.id === params.target); const sourceNode = nodes.find(n => n.id === params.source); + if (!targetNode || !sourceNode) { + return; + } const targetHandle = getHandle(targetNode, params.targetHandle, true); const sourceHandle = getHandle(sourceNode, params.sourceHandle, false); const edge = { @@ -153,12 +164,12 @@ export default function Flow({ filename }) { if (n.parentId) { const parent = deletedNodesMap[n.parentId]; if (parent) { - n.parentId = null; + n.parentId = undefined; n.position = { x: n.position.x + parent.position.x, y: n.position.y + parent.position.y }; } } }); - }); + }, []); useEffect(() => { if (graphStore.current) { @@ -173,12 +184,12 @@ export default function Flow({ filename }) { if (!event) { return; } - if (event.type === 'dblclick' && !isAddNodeActive) { + if (event.type === 'dblclick' && !isAddNodeActive && reactFlowInstance.current) { // setIsAddNodeActive(true); setEventMousePos({ x: event.clientX, y: event.clientY }); setNodeToPos(reactFlowInstance.current.screenToFlowPosition({ x: event.clientX, y: event.clientY })); } - }); + }, []); const handleMouseClick = useCallback((event) => { if (event.type === 'click') { @@ -195,6 +206,9 @@ export default function Flow({ filename }) { }, [handleMouseClick]); const onDrop = useCallback((event) => { + if (!reactFlowInstance.current || !API) { + return; + } filesystemDragEnd(reactFlowInstance.current, API, event); }, [reactFlowInstance, API]); @@ -218,9 +232,15 @@ export default function Flow({ filename }) { }, []); const isValidConnection = useCallback((connection) => { + if (!reactFlowInstance.current) { + return false; + } const { getNode, getNodes, getEdges } = reactFlowInstance.current; const srcNode = getNode(connection.source); const tgtNode = getNode(connection.target); + if (!srcNode || !tgtNode) { + return false; + } const srcHandle = getHandle(srcNode, connection.sourceHandle, false); const tgtHandle = getHandle(tgtNode, connection.targetHandle, true); @@ -350,7 +370,6 @@ export default function Flow({ filename }) {
- - {/* {filename} */} - {nodeMenu && } - {paneMenu && setPaneMenu(null)} {...paneMenu} />} - - + {paneMenu && setPaneMenu(null)} top={paneMenu.top} left={paneMenu.left} />} + + {isAddNodeActive && }
@@ -402,6 +418,9 @@ function ControlRow() { const filename = useFilename(); const run = useCallback(async () => { + if (!API) { + return; + } const [[graph, resources], errors] = await Graph.serializeForAPI(nodes, edges); if (errors.length > 0) { notification.error({ @@ -417,11 +436,17 @@ function ControlRow() { }, [API, nodes, edges, notification, filename]); const pause = useCallback(() => { + if (!API) { + return; + } API.pause(); runStateShouldChange(); }, [API]); const clear = useCallback(async () => { + if (!API) { + return; + } const [[graph, resources], errors] = await Graph.serializeForAPI(nodes, edges); if (errors.length > 0) { notification.error({ @@ -443,7 +468,7 @@ function ControlRow() { runState !== 'stopped' ? (
- - - {contextMenu && ( - +
+ + + {filesRoot}/ +
+
+ + - )} -
+ {contextMenu && ( + + )} + + + ); } diff --git a/web/src/components/TopPanel.tsx b/web/src/components/TopPanel.tsx index f01e0b5..8e31b83 100644 --- a/web/src/components/TopPanel.tsx +++ b/web/src/components/TopPanel.tsx @@ -1,11 +1,11 @@ import React, { useCallback, useEffect, useState, useMemo } from 'react'; -import { Popover, Input, Modal, Typography, Space, theme } from 'antd'; +import { Popover, Input, Modal, Typography, Space, Popconfirm, theme } from 'antd'; import { LinkOutlined, DisconnectOutlined, SettingFilled } from '@ant-design/icons'; import Settings from './Settings'; import { useSettings } from '../hooks/Settings'; -import { useAPI, useAPIMessage } from '../hooks/API'; +import { useAPI, useAPIMessage, useAPIReconnectTimer } from '../hooks/API'; import { SparklineChart, LinearYAxis, LinearYAxisTickSeries, LinearXAxis, LinearXAxisTickSeries, LineSeries, TooltipArea, ChartTooltip, StackedBarChart, StackedBarSeries, Bar } from 'reaviz'; import type { ChartShallowDataShape, ChartNestedDataShape } from 'reaviz'; import './top-panel.css'; @@ -17,53 +17,31 @@ const disconnectedStyle = { ...iconStyle, color: 'red' }; export default function TopPanel() { const title = 'Graphbook'; - const [settings, setSetting] = useSettings(); const [connected, setConnected] = useState(false); - const [host, setHost] = useState(settings.graphServerHost); - const [hostEditorOpen, setHostEditorOpen] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false); + const [popconfirmOpen, setPopconfirmOpen] = useState(false); + const reconnectTime = useAPIReconnectTimer(); const API = useAPI(); useEffect(() => { if (!API) { setConnected(false); + setPopconfirmOpen(true); } else { setConnected(true); + setPopconfirmOpen(false); } }, [API]); - const setAPIHost = useCallback(host => { - setSetting("graphServerHost", host); - }, []); - - const onHostChange = useCallback((e) => { - setHost(e.target.value); - }, [host]); - - const onOpenChange = useCallback(isOpen => { - setHostEditorOpen(isOpen); - if (!isOpen) { - setAPIHost(host); - } - }, [host]); - - const closeAndUpdateHost = useCallback(() => { - setHostEditorOpen(false); - setAPIHost(host); - }, []); const setSettingsModal = useCallback((shouldOpen) => { setSettingsOpen(shouldOpen); }, []); - const hostEditor = ( - - ); + const closePopconfirm = useCallback(() => { + setPopconfirmOpen(false); + }, []); + return (
@@ -78,9 +56,20 @@ export default function TopPanel() { setSettingsModal(true)} /> - - {connected ? : } - + {connected ? + : + + + + }
@@ -254,7 +243,7 @@ function WorkerChart() { const enqueuedSize = queueSizes.dump.reduce((acc, val) => acc + val, 0) + queueSizes.load.reduce((acc, val) => acc + val, 0); - const outgoingSize = + const outgoingSize = queueSizes.load_result.reduce((acc, val) => acc + val, 0) + queueSizes.dump_result.reduce((acc, val) => acc + val, 0) + queueSizes.total_consumer_size; @@ -274,7 +263,7 @@ function WorkerChart() { if (newData.length > SPARK_SIZE_LIMIT) { newData.shift(); } - return newData; + return newData; } return prev; @@ -298,7 +287,7 @@ function WorkerChart() { const XAxis = useMemo(() => { return ( - } /> + } /> ); }, []); diff --git a/web/src/hooks/API.ts b/web/src/hooks/API.ts index 8b6d246..2581793 100644 --- a/web/src/hooks/API.ts +++ b/web/src/hooks/API.ts @@ -9,16 +9,27 @@ let initialized = false; const initialize = () => setGlobalAPI(API); const disable = () => setGlobalAPI(null); +function setGlobalAPI(api: ServerAPI | null) { + globalAPI = api; + for (const setter of localSetters) { + setter(globalAPI); + } +} + export function useAPI() { const [_, setAPI] = useState(globalAPI); useEffect(() => { + localSetters.push(setAPI); + if (!initialized) { API.addWsEventListener('open', initialize); API.addWsEventListener('close', disable); initialized = true; } return () => { + localSetters = localSetters.filter((setter) => setter !== setAPI); + if (localSetters.length === 0) { API.removeWsEventListener('open', initialize); API.removeWsEventListener('close', disable); @@ -27,23 +38,9 @@ export function useAPI() { } }, []); - useEffect(() => { - localSetters.push(setAPI); - return () => { - localSetters = localSetters.filter((setter) => setter !== setAPI); - }; - }, []); - return globalAPI; } -function setGlobalAPI(api: ServerAPI | null) { - globalAPI = api; - for (const setter of localSetters) { - setter(globalAPI); - } -} - let apiMessageCallbacks: { [event_type: string]: Function[] } = {}; let apiSetTo: ServerAPI | null = null; export function useAPIMessage(event_type: string, callback: Function) { @@ -81,3 +78,37 @@ export function useAPINodeMessage(event_type: string, node_id: string, filename: } }, [node_id, callback, filename])); } + + +let localReconnectListeners: Function[] = []; +let reconnectInitialized = false; +let timeUntilReconnect = 0; +export function useAPIReconnectTimer() { + const [_, reconnectTime] = useState(timeUntilReconnect); + + const onTimerChanged = (time: number) => { + timeUntilReconnect = time; + for (const listener of localReconnectListeners) { + listener(time); + } + }; + + useEffect(() => { + localReconnectListeners.push(reconnectTime); + + if(!reconnectInitialized) { + API.addReconnectListener(onTimerChanged); + reconnectInitialized = true; + } + + return () => { + localReconnectListeners = localReconnectListeners.filter((listener) => listener !== reconnectTime); + if(localReconnectListeners.length === 0) { + API.removeReconnectListener(onTimerChanged); + reconnectInitialized = false; + } + } + }, []); + + return timeUntilReconnect; +} From 06ad8eb9cc64c66f08a3dfec4a44f56a37bb643b Mon Sep 17 00:00:00 2001 From: "Richard S. Franklin" Date: Wed, 7 Aug 2024 11:24:52 -0700 Subject: [PATCH 15/21] Copying outputs if flag enabled (#68) --- graphbook/processing.py | 5 ++++- graphbook/server.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/graphbook/processing.py b/graphbook/processing.py index 22f71ed..ceb5209 100644 --- a/graphbook/processing.py +++ b/graphbook/processing.py @@ -11,6 +11,7 @@ import traceback import asyncio import time +import copy class WebInstanceProcessor: @@ -21,6 +22,7 @@ def __init__( view_manager_queue: mp.Queue, output_dir: str, continue_on_failure: bool, + copy_outputs: bool, custom_nodes_path: str, close_event: mp.Event, pause_event: mp.Event, @@ -33,6 +35,7 @@ def __init__( self.graph_state = GraphState(custom_nodes_path, view_manager_queue) self.output_dir = output_dir self.continue_on_failure = continue_on_failure + self.copy_outputs = copy_outputs self.custom_nodes_path = custom_nodes_path self.num_workers = num_workers self.steps = {} @@ -63,7 +66,7 @@ def exec_step( return None if outputs is not None: - self.graph_state.handle_outputs(step.id, outputs) + self.graph_state.handle_outputs(step.id, outputs if not self.copy_outputs else copy.deepcopy(outputs)) self.view_manager.handle_outputs(step.id, outputs) self.view_manager.handle_time(step.id, time.time() - start_time) return outputs diff --git a/graphbook/server.py b/graphbook/server.py index 1c07bb6..16f9d96 100644 --- a/graphbook/server.py +++ b/graphbook/server.py @@ -394,6 +394,7 @@ def get_args(): parser.add_argument("--nodes_dir", type=str, default="./workflow/custom_nodes") parser.add_argument("--num_workers", type=int, default=1) parser.add_argument("--continue_on_failure", action="store_true") + parser.add_argument("--copy_outputs", action="store_true") return parser.parse_args() @@ -488,6 +489,7 @@ async def start(): view_manager_queue, args.output_dir, args.continue_on_failure, + args.copy_outputs, custom_nodes_path, close_event, pause_event, From 19fc3f32e122c36a9a45692045b7814e03d4ca6e Mon Sep 17 00:00:00 2001 From: "Richard S. Franklin" Date: Wed, 21 Aug 2024 17:15:43 -0700 Subject: [PATCH 16/21] Refactors and docs (#71) --- Dockerfile | 2 +- README.md | 6 +- docs/_static/concepts/graphbookworkers.svg | 4 + .../concepts/graphbookworkersgraph.svg | 4 + docs/_static/concepts/workers-vis.png | Bin 0 -> 2379 bytes docs/concepts.rst | 71 +++++++++ graphbook/main.py | 38 +++++ graphbook/media.py | 15 +- .../web_processor.py} | 42 +++--- graphbook/utils.py | 2 + graphbook/{server.py => web.py} | 63 ++------ poetry.lock | 20 ++- pyproject.toml | 1 + scripts/graphbook | 2 +- tests/test_processor.py | 138 ------------------ tests/test_steps.py | 18 --- web/src/components/Flow.tsx | 1 + web/src/components/Nodes/Node.tsx | 123 +++++++++++----- web/src/components/Nodes/node.css | 4 + web/src/components/Settings.tsx | 16 +- web/src/hooks/Settings.ts | 2 + 21 files changed, 300 insertions(+), 272 deletions(-) create mode 100644 docs/_static/concepts/graphbookworkers.svg create mode 100644 docs/_static/concepts/graphbookworkersgraph.svg create mode 100644 docs/_static/concepts/workers-vis.png create mode 100644 graphbook/main.py rename graphbook/{processing.py => processing/web_processor.py} (90%) rename graphbook/{server.py => web.py} (88%) delete mode 100644 tests/test_processor.py delete mode 100644 tests/test_steps.py diff --git a/Dockerfile b/Dockerfile index 87cd6d7..9351ee4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,4 +39,4 @@ RUN make web EXPOSE 8005 8006 8007 -CMD ["python", "graphbook/server.py", "--web", "--web_dir", "web/dist"] +CMD ["python", "graphbook/main.py", "--web", "--web_dir", "web/dist"] diff --git a/README.md b/README.md index 8a29bb1..ca04225 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ Graphbook is in a very early stage of development, so expect minor bugs and rapi - Autosaving and shareable serialized workflow files - Registers node code changes without needing a restart - Monitorable CPU and GPU resource usage +- (BETA) Remote subgraphs for scaling workflows on other Graphbook services ### Planned Features -- Graphbook services and remote DAGs for scalable workflows - A `graphbook run` command to execute workflows in a CLI -- Step/Resource functions to reduce verbosity +- Step/Resource functions with decorators to reduce verbosity - Human-in-the-loop Steps for manual feedback/control during DAG execution - All-code workflows, so users never have to leave their IDE - UI extensibility @@ -80,7 +80,7 @@ You can use any other virtual environment solution, but it is highly adviced to 1. Clone the repo and `cd graphbook` 1. `poetry install --with dev` 1. `poetry shell` -1. `python graphbook/server.py` +1. `python graphbook/main.py` 1. `cd web` 1. `npm install` 1. `npm run dev` diff --git a/docs/_static/concepts/graphbookworkers.svg b/docs/_static/concepts/graphbookworkers.svg new file mode 100644 index 0000000..010aba8 --- /dev/null +++ b/docs/_static/concepts/graphbookworkers.svg @@ -0,0 +1,4 @@ + + + +
load
dump
Workers
Work Queues
Result Queues
Nodes
C
X
Y
Consumer Queues
Graph Execution
1
2
3
4
5
6
 \ No newline at end of file diff --git a/docs/_static/concepts/graphbookworkersgraph.svg b/docs/_static/concepts/graphbookworkersgraph.svg new file mode 100644 index 0000000..e74541b --- /dev/null +++ b/docs/_static/concepts/graphbookworkersgraph.svg @@ -0,0 +1,4 @@ + + + +
B
C
A
X
Y
Graph Execution
 \ No newline at end of file diff --git a/docs/_static/concepts/workers-vis.png b/docs/_static/concepts/workers-vis.png new file mode 100644 index 0000000000000000000000000000000000000000..d57f82ce34122cc46c1ab8fd09b57b452aaafaf8 GIT binary patch literal 2379 zcmV-R3AFZ!P)ZgXgFbngSdJ^%m!Ep$a#bVG7w zVRUJ4ZXi@?ZDjycb#5RrI3PiFXCN{#GBhADFgh|dIyEsMP)#6EL{ujrB-{W12&73w zK~#90?VNv1l=mISU&n!uobBVndbB2eQX0?R%2_s!of%Ic@me!c8?huUVd+rUqQqY~ zgOCO!3@}E#wNbmq)*38pYKRGiCUkV^JQJITZ45T5*FeV)(vll#8;e7?`~d=Jye$jAr=20Dm~bpQqn7ehP-x-rCKpc_Lx z2D*ulbpZdb^!E0~I)oYV#1l_U+h>R;hTLvq<^y*UxFMc-hC2KW#|BO@IPx8fObGx~ zKDZZcWDsp+5P(FJKtiN1mKfrhXNYkR^8Sqv$Jr_$+>2mZgxvQm09_CNJ{F-(!Vu2_ z62vcH+X93gm-DdzS zkAI8@OiS+h_=b2EFu&;E3&gH`8REGQ_(c0N08b_SfTeNbJsZ~$&wW72zbk=zpMi{b zTL!w(B_8d3IWEUmn)=6C@$Pq#^2|ZHL(z6WuWz)orrsBBhJq-ktJu0Rn>2?VyTiqr z=ZiVm>_d(A)>eW&C54pe!=6U=#$r4l1<<0;bCu(b9#iec^IRI)`&tQvfRK;Z*H7>@ zcG$@~6#zh~c1`NHr@7d$HKGZ2oTU8cYjCC6NwL{U%i6%!+9rGhdcM(xiLSCG zINiuX8A_){{#qey4EazxHAJ-w<#TW@kDUCaf^sDp_bUzv1!&##Gs?Op3U}79+a)1} z`tV<9;&^o)im!n)2V};oc2X`!Kd3}DN7`B+T_1Qjux~58+eT@+FgZBQ$H|@BX-%%6 z;Yco$83+LgICeB};st5aC5x@MY{Kdq+77K_X{7z_wM(@Bri!wcRKyPt;}(G{C)vKe zk)=CpI8&d2v_zxN_d7h*RTQhqv>kCX+m(kp{2%&;gR{IUw&+{)h#AOE4bO!xw6!t< zz~yr&tYcH=Ih2kdyOvu4P&*YaS!9ZwptgCaJ8xxoa|65VqZ4n#mMSCL%h|BKisR3I zOhtO6%e)wInHB4(c(9VQrvkrs11(-ZrRk3GafJdj?JMRj^EOT&%g1IO-z|J!LQ0t| zp7AFwC3%_k*5og9Q&hVPCEJ6y2i)19_9*z0ayav{du$9Tvyl4Nl4%Jf&+^L*BgQ>| zJD8{I+E}4;!E8m&%0T_Cf-fZFG((`Ho4&Ms3Y`H8Iuz6wa}m+#y3hrgwPXky%Ew-` z)wLAbCuaj)>Po25jKmO%!;|&rBgP<+C^*PE!`&8$3baY}$2){5LVu(KB zYvP7Lx@G}j&|<+9Kl3S15-qpw*L}AoWy;u&oQsqw6{X`6D^?%H<_eJfwnCSdi#tAC z$})D!hy(}(G^A|XG%4MN<8guGR|9BFBc70ozu806rxNZmTa^7e%AqYZzORwrP=s^J z=Vcn*^=q)dI>9WssXu>$-0AAfK!EmF8&SlkIq}1YExnYZ)hcE6dAJTqI9wUXt7M*D zn}hSw*{k;P{bc}*z78FH>M26NXvIyucyUVm#LvWw6=Ob!uH+DecjW+_=HeKLUil_$Q_%76=XVw6p@V^`RL=GBy9Z{XAv|a&|I}9SZ95 zF1p1GHm8K`8P4mx)!`=~D)^Vn$d3X@*dzh}Wj`v8DYB0r-`_Q)OdA4`5|NZk&Dqk( zf;m9b{$ko4rBpm`8$UToo2Yqn7w-66gsXX!?Kz2bdUENS$(l#VETy?NCmi3kcB-~D zB5tpwD1D05WEQcxaV-yHdTr$7 zL@k2Ni%CuiPq>sKv-?-u5Sy#;Tu{efY$p6?V3x=~bd*i~9!jdc=(=~YBuB=RkkX5& zDY4wLgfzvlUPEJNTBb!)=?^-K;SsCX%>5hb09sU`kA! zJI8at-92FzW!!0f_^+-+c15LiNk&fc;~$c7k6ZgnJD&D$ zaaT9fT;{yP^AQ2Z&RU9uCaMm3nNB}KY7s|k-2`4O=HU6!voyMo<&mA0J?Y5W*D|dy z%o2swyIFCejss`I@{wLV%9(1J>#YabvNnsfRaxZy@-2Li?WW<>u1E&H|1lXqH;xUx zy}e@_;71q4qde@LAKOf$W1I@(-IjrF4DlG~#t@HzZVd4l=*AF_fo_bQg!2d?OD4`@ zVOUj(6Yt%2QoR`BnMZuDwmQ}!%nGAy8R*7%NEjpThkvnRh{r%TM%gmZP5eShp9b>} x)4;%h?y{vuHm-#vSpx~{{lb71LD|bXLbMp002ovPDHLkV1k4uedYiF literal 0 HcmV?d00001 diff --git a/docs/concepts.rst b/docs/concepts.rst index 7c2d6a5..16c5e30 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -30,3 +30,74 @@ A Step goes through lifecycle methods upon processing a Note. The below methods #. ``forward_note``: This is called after the Step processes each Note and is used to route the Note to a certain output slot. #. ``on_end``: This is called at the end of each graph execution. +Workers +******** + +In order to maximize the utilization of the GPU during graph execution, we parallelize the preparation of inputs and outputs +for each BatchStep (an extension of Step) across a number of workers. +A BatchStep can require inputs to be prepared and outputs to be saved by workers. +Each worker is a separate process that can run in parallel with others. +Each worker has its own work queue and result queue for incoming work and outgoing results, respectively. +A worker is dedicated to either preparing inputs or saving outputs, but not both. Whether it is preparing inputs or saving outputs, the worker logic +is relatively the same. +The Graphbook worker implementation also accounts for graph changes. +In between graph executions, nodes can be added, removed, or modified, so the workers must adapt to these changes. + +Logic Details +============= + +.. image:: _static/concepts/graphbookworkersgraph.svg + :alt: Example Graph + :align: center + +The above graph is used to explain the worker logic. + +.. image:: _static/concepts/graphbookworkers.svg + :alt: Graphbook Worker Concepts Illustration + :align: center + + +The logic behind the workers is detailed in the following steps (1-6): + +#. + A BatchStep prepares the item's parameter inputs and its belonging Note ID and Step ID. The IDs are Python's unique identifiers for each object taken with ``id()``. + We supply the workers with this information, so that when they are done processing the item, they can be matched later with the correct Note and Step. + The actual function, implemented by the BatchStep, is stored inside of a shared dictionary that the workers can access later. +#. + A BatchStep enqueues the item in one of the load and dump queues, so that the workers can access them. +#. + The workers will then dequeue the work from their work queues and execute the corresponding BatchStep's function (``load_fn()`` and ``dump_fn()``) on the item if the BatchStep still exists, but before they do that, they need to check the size of the result queue. + If the result queue is full, the worker will block until space is available. +#. + After the worker has finished processing the item, it will enqueue the result in the result queue. The workers cannot deliver the results directly to the consumer queues because they are provisioned + dynamically by the main process since the graph can have different nodes in between different executions. +#. + The main process will then dequeue the results from the result queues and deliver them to the correct corresponding consumer queues as long as the sum of the sizes of all consumer queues is less than a certain limit and if the consumer node still exists. +#. + The consumer nodes will then dequeue the results from their consumer queues and process them in their correct lifecycle method. + Completed load items will be delivered to ``on_item_batch(results: List[any], items: List[any], notes: List[Note])`` where results, items, and notes are in order; i.e. ``results[i]`` corresponds to input ``items[i]`` and belonging to note ``notes[i]``. + The size of the results, items, and notes lists will be equal to the batch size (or less if the batch size is not met). + Completed dumped items will not be delivered to any lifecycle method. + However, the BatchStep will still search for completed dumped items and keep track of which Note they belong to. + If all dumped items from a Note are completed, then the Note is considered finished and can be delivered to the next Step for processing. + We do this because if a following Step depends on the saving of a particular item from that Note, then that Step will execute too soon. + +Worker Performance Visualization +================================================= + +Sometimes, we do not know exactly how many workers will be needed. For this reason, Graphbook will offer an auto-scaling feature that will automatically adjust the number of workers based on the workload. +For now, Graphbook offers a visualization about the performance of the workers that can indicate to the user when there are too many or too few workers, so that they can manually adjust the number of workers that they need. +See example below: + + +.. image:: _static/concepts/workers-vis.png + :alt: Graphbook Worker Performance Visualization + :align: center + + +The visualization is in the form of a centered bar chart that shows the number of items that are enqueued in the work queues as red bars and the number of items that are in the result and consumer queues as green bars. Refer to the following when reading this chart: + +#. If the red bars are consistently longer than the green bars and there's hardly any green, it indicates that there are too few workers. +#. If the red bars are consistently longer than the green bars but there is some green, then it indicates that the graph execution on the main process is just too slow to consume all of the results which, in turn, creates a conjestion in the workers work queues. This is because the result queues have a max size, and if they are full, the workers will be blocked until space is available while the work queues are being loaded. A max size per result queue is enforced to help prevent memory overloading issues. +#. If the green bars are consistently longer than the red bars, it indicates there may be enough or too many workers dependending on your system constraints. +#. If there are no visible bars, it indicates that the workers are not being utilized. diff --git a/graphbook/main.py b/graphbook/main.py new file mode 100644 index 0000000..a0a174c --- /dev/null +++ b/graphbook/main.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +import argparse +from graphbook.web import start_web + +DESCRIPTION = """ +Graphbook | ML Workflow Framework +""" + + +def get_args(): + parser = argparse.ArgumentParser( + description=DESCRIPTION, formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument("--num_workers", type=int, default=1) + parser.add_argument("--continue_on_failure", action="store_true") + parser.add_argument("--copy_outputs", action="store_true") + + # Web subcommand + parser.add_argument("--media_dir", type=str, default="/") + parser.add_argument("--web_dir", type=str) + parser.add_argument("--host", type=str, default="0.0.0.0") + parser.add_argument("--graph_port", type=int, default=8005) + parser.add_argument("--media_port", type=int, default=8006) + parser.add_argument("--web_port", type=int, default=8007) + parser.add_argument("--workflow_dir", type=str, default="./workflow") + parser.add_argument("--nodes_dir", type=str, default="./workflow/custom_nodes") + + return parser.parse_args() + + +def main(): + args = get_args() + + start_web(args) + + +if __name__ == "__main__": + main() diff --git a/graphbook/media.py b/graphbook/media.py index d556586..3401f5d 100644 --- a/graphbook/media.py +++ b/graphbook/media.py @@ -21,11 +21,11 @@ async def cors_middleware(request: web.Request, handler): class MediaServer: def __init__( self, - address="0.0.0.0", + host="0.0.0.0", port=8006, root_path="./workflow", ): - self.address = address + self.host = host self.port = port self.root_path = root_path routes = web.RouteTableDef() @@ -57,14 +57,21 @@ async def handle(request: web.Request) -> web.Response: async def _async_start(self): runner = web.AppRunner(self.app) await runner.setup() - site = web.TCPSite(runner, self.address, self.port, shutdown_timeout=0.5) + site = web.TCPSite(runner, self.host, self.port, shutdown_timeout=0.5) await site.start() await asyncio.Event().wait() def start(self): self.app.router.add_routes(self.routes) - print(f"Starting media server at {self.address}:{self.port}") + print(f"Starting media server at {self.host}:{self.port}") try: asyncio.run(self._async_start()) except KeyboardInterrupt: print("Exiting media server") + + +def create_media_server(args): + server = MediaServer( + host=args.host, port=args.media_port, root_path=args.media_dir + ) + server.start() diff --git a/graphbook/processing.py b/graphbook/processing/web_processor.py similarity index 90% rename from graphbook/processing.py rename to graphbook/processing/web_processor.py index ceb5209..9a10df9 100644 --- a/graphbook/processing.py +++ b/graphbook/processing/web_processor.py @@ -1,7 +1,7 @@ from graphbook.steps import Step, SourceStep, AsyncStep, StepOutput from graphbook.dataloading import Dataloader -from .note import Note -from typing import List +from ..note import Note +from typing import List, Dict import queue import multiprocessing as mp import multiprocessing.connection as mpc @@ -20,7 +20,6 @@ def __init__( cmd_queue: mp.Queue, server_request_conn: mpc.Connection, view_manager_queue: mp.Queue, - output_dir: str, continue_on_failure: bool, copy_outputs: bool, custom_nodes_path: str, @@ -33,7 +32,6 @@ def __init__( self.pause_event = pause_event self.view_manager = ViewManagerInterface(view_manager_queue) self.graph_state = GraphState(custom_nodes_path, view_manager_queue) - self.output_dir = output_dir self.continue_on_failure = continue_on_failure self.copy_outputs = copy_outputs self.custom_nodes_path = custom_nodes_path @@ -43,6 +41,7 @@ def __init__( self.state_client = ProcessorStateClient( server_request_conn, close_event, self.graph_state, self.dataloader ) + self.remote_subgraphs: Dict[str, NetworkClient] = {} self.is_running = False self.filename = None @@ -66,7 +65,9 @@ def exec_step( return None if outputs is not None: - self.graph_state.handle_outputs(step.id, outputs if not self.copy_outputs else copy.deepcopy(outputs)) + self.graph_state.handle_outputs( + step.id, outputs if not self.copy_outputs else copy.deepcopy(outputs) + ) self.view_manager.handle_outputs(step.id, outputs) self.view_manager.handle_time(step.id, time.time() - start_time) return outputs @@ -180,10 +181,10 @@ def setup_dataloader(self, steps: List[Step]): for c in dataloader_consumers: c.set_dataloader(self.dataloader) - def try_update_state(self, queue_entry: dict) -> bool: + def try_update_state(self, local_graph: dict) -> bool: try: self.graph_state.update_state( - queue_entry["graph"], queue_entry["resources"] + local_graph["graph"], local_graph["resources"] ) return True except NodeInstantiationError as e: @@ -193,25 +194,28 @@ def try_update_state(self, queue_entry: dict) -> bool: traceback.print_exc() return False + def exec(self, work: dict): + self.set_is_running(True, work["filename"]) + if not self.try_update_state(work): + return + + if work["cmd"] == "run_all": + self.run() + elif work["cmd"] == "run": + self.run(work["step_id"]) + elif work["cmd"] == "step": + self.step(work["step_id"]) + async def start_loop(self): loop = asyncio.get_running_loop() loop.run_in_executor(None, self.state_client.start) + exec_cmds = ["run_all", "run", "step"] while not self.close_event.is_set(): self.set_is_running(False) try: work = self.cmd_queue.get(timeout=MP_WORKER_TIMEOUT) - if work["cmd"] == "run_all": - self.set_is_running(True, work["filename"]) - if self.try_update_state(work): - self.run() - elif work["cmd"] == "run": - self.set_is_running(True, work["filename"]) - if self.try_update_state(work): - self.run(work["step_id"]) - elif work["cmd"] == "step": - self.set_is_running(True, work["filename"]) - if self.try_update_state(work): - self.step(work["step_id"]) + if work["cmd"] in exec_cmds: + self.exec(work) elif work["cmd"] == "clear": self.graph_state.clear_outputs(work.get("node_id")) self.view_manager.handle_clear(work.get("node_id")) diff --git a/graphbook/utils.py b/graphbook/utils.py index d3e2c0f..2eaf2ed 100644 --- a/graphbook/utils.py +++ b/graphbook/utils.py @@ -7,6 +7,8 @@ import os import platform import multiprocessing.connection as mpc +import pickle +from asyncio.streams import StreamWriter MP_WORKER_TIMEOUT = 5.0 diff --git a/graphbook/server.py b/graphbook/web.py similarity index 88% rename from graphbook/server.py rename to graphbook/web.py index 16f9d96..7e2a41c 100644 --- a/graphbook/server.py +++ b/graphbook/web.py @@ -1,6 +1,6 @@ import aiohttp from aiohttp import web -from graphbook.processing import WebInstanceProcessor +from graphbook.processing.web_processor import WebInstanceProcessor from graphbook.viewer import ViewManager from graphbook.exports import NodeHub import os, sys @@ -13,10 +13,9 @@ import multiprocessing.connection as mpc import asyncio import base64 -import argparse import hashlib from graphbook.state import UIState -from graphbook.media import MediaServer +from graphbook.media import create_media_server from graphbook.utils import poll_conn_for, ProcessorStateRequest try: import magic @@ -50,12 +49,12 @@ def __init__( processor_pause_event: mp.Event, view_manager_queue: mp.Queue, close_event: mp.Event, - address="0.0.0.0", + host="0.0.0.0", port=8005, root_path="./workflow", custom_nodes_path="./workflow/custom_nodes", ): - self.address = address + self.host = host self.port = port self.node_hub = NodeHub(custom_nodes_path) self.ui_state = None @@ -160,14 +159,9 @@ async def pause(request: web.Request) -> web.Response: @routes.post("/clear/{id}") async def clear(request: web.Request) -> web.Response: node_id = request.match_info.get("id") - data = await request.json() - graph = data.get("graph", {}) - resources = data.get("resources", {}) processor_queue.put( { "cmd": "clear", - "graph": graph, - "resources": resources, "node_id": node_id, } ) @@ -339,7 +333,7 @@ def delete(request): async def _async_start(self): runner = web.AppRunner(self.app) await runner.setup() - site = web.TCPSite(runner, self.address, self.port, shutdown_timeout=0.5) + site = web.TCPSite(runner, self.host, self.port, shutdown_timeout=0.5) loop = asyncio.get_running_loop() await site.start() await loop.run_in_executor(None, self.view_manager.start) @@ -347,7 +341,7 @@ async def _async_start(self): def start(self): self.app.router.add_routes(self.routes) - print(f"Starting graph server at {self.address}:{self.port}") + print(f"Starting graph server at {self.host}:{self.port}") self.node_hub.start() try: asyncio.run(self._async_start()) @@ -357,8 +351,8 @@ def start(self): class WebServer: - def __init__(self, address, port, web_dir): - self.address = address + def __init__(self, host, port, web_dir): + self.host = host self.port = port if web_dir is None: web_dir = osp.join(osp.dirname(__file__), "web") @@ -372,32 +366,14 @@ def start(self): ) return os.chdir(self.cwd) - with socketserver.TCPServer((self.address, self.port), self.server) as httpd: - print(f"Starting web server at {self.address}:{self.port}") + with socketserver.TCPServer((self.host, self.port), self.server) as httpd: + print(f"Starting web server at {self.host}:{self.port}") print(f"Visit http://127.0.0.1:{self.port}") try: httpd.serve_forever() except KeyboardInterrupt: print("Exiting web server") - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument("--output_dir", type=str, default="./output") - parser.add_argument("--media_dir", type=str, default="/") - parser.add_argument("--web_dir", type=str) - parser.add_argument("--address", type=str, default="0.0.0.0") - parser.add_argument("--graph_port", type=int, default=8005) - parser.add_argument("--media_port", type=int, default=8006) - parser.add_argument("--web_port", type=int, default=8007) - parser.add_argument("--workflow_dir", type=str, default="./workflow") - parser.add_argument("--nodes_dir", type=str, default="./workflow/custom_nodes") - parser.add_argument("--num_workers", type=int, default=1) - parser.add_argument("--continue_on_failure", action="store_true") - parser.add_argument("--copy_outputs", action="store_true") - return parser.parse_args() - - def create_graph_server( args, cmd_queue, @@ -416,26 +392,18 @@ def create_graph_server( close_event, root_path=root_path, custom_nodes_path=custom_nodes_path, - address=args.address, + host=args.host, port=args.graph_port, ) server.start() -def create_media_server(args): - server = MediaServer( - address=args.address, port=args.media_port, root_path=args.media_dir - ) - server.start() - - def create_web_server(args): - server = WebServer(address=args.address, port=args.web_port, web_dir=args.web_dir) + server = WebServer(host=args.host, port=args.web_port, web_dir=args.web_dir) server.start() -def main(): - args = get_args() +def start_web(args): cmd_queue = mp.Queue() parent_conn, child_conn = mp.Pipe() view_manager_queue = mp.Queue() @@ -487,7 +455,6 @@ async def start(): cmd_queue, parent_conn, view_manager_queue, - args.output_dir, args.continue_on_failure, args.copy_outputs, custom_nodes_path, @@ -498,7 +465,3 @@ async def start(): await processor.start_loop() asyncio.run(start()) - - -if __name__ == "__main__": - main() diff --git a/poetry.lock b/poetry.lock index 6f23ebe..0b3025d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1170,6 +1170,24 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.23.8" +description = "Pytest support for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest_asyncio-0.23.8-py3-none-any.whl", hash = "sha256:50265d892689a5faefb84df80819d1ecef566eb3549cf915dfb33569359d1ce2"}, + {file = "pytest_asyncio-0.23.8.tar.gz", hash = "sha256:759b10b33a6dc61cce40a8bd5205e302978bbbcc00e279a8b61d9a6a3c82e4d3"}, +] + +[package.dependencies] +pytest = ">=7.0.0,<9" + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "python-magic" version = "0.4.27" @@ -2183,4 +2201,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "80919b9f04264781b972188b52cdd141714992ab7d1d8c7ec9b7ab4d76bcc801" +content-hash = "eae7ecdee218a49478ecbda7b95c22a49aa490fac7163cf74700ea3be40139ab" diff --git a/pyproject.toml b/pyproject.toml index 03ade44..2c4cec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ sphinxcontrib-jquery = "^4.1" sphinxcontrib-jsmath = "^1.0.1" sphinxcontrib-qthelp = "^1.0.7" sphinxcontrib-serializinghtml = "^1.1.10" +pytest-asyncio = "^0.23.8" [tool.poetry.group.examples] optional = true diff --git a/scripts/graphbook b/scripts/graphbook index cdc4b4f..b984229 100755 --- a/scripts/graphbook +++ b/scripts/graphbook @@ -2,7 +2,7 @@ REPO_PATH=$(dirname $(realpath $0))/.. SCRIPT_DIR=$0 -SERVER_PATH=$REPO_PATH/graphbook/server.py +SERVER_PATH=$REPO_PATH/graphbook/main.py WEB_PATH=$REPO_PATH/web WEB_BUILD_PATH=$WEB_PATH/dist diff --git a/tests/test_processor.py b/tests/test_processor.py deleted file mode 100644 index 2b73b5d..0000000 --- a/tests/test_processor.py +++ /dev/null @@ -1,138 +0,0 @@ -from graphbook.processor import Sequential, TreeProcessor, Print, Copy, LoadJSONL, Step, Note, SplitRecordsByItems, StepOutput, StepData, BatchStep -from arithmetic import SumByConstant, DivByConstant, MulByConstant, NumNote, NumListNote -from typing import List -import os - -class AssertStep(Step): - def __init__(self, assertions: List): - super().__init__(item_key="num") - self.assertions = assertions - self.current_assertion = 0 - - def exec(self, data: StepData) -> StepOutput: - items, _, completed = data - assert self.assertions[self.current_assertion](items) - self.current_assertion = (self.current_assertion + 1) % len(self.assertions) - return { - "next": completed - } - -def test_single_step(): - steps = AssertStep([lambda items: items[0].item == 1]) - TreeProcessor(steps).run( [NumNote("num", 1)] ) - -def test_split(): - a_step = Sequential([ - MulByConstant(10), - AssertStep([lambda items: items[0].item == 10]) - ]) - b_step = Sequential([ - DivByConstant(10), - AssertStep([lambda items: items[0].item == 1]) - ]) - steps = SplitRecordsByItems( - lambda items: items[0].item < 5, - item_key="num", - a_step=a_step, - b_step=b_step - ) - - TreeProcessor(steps).run( [NumNote("num", 1), NumNote("num", 10)] ) - -def test_sequential(): - steps = Sequential([ - SumByConstant(5), - DivByConstant(2), - AssertStep([lambda items: items[0].item == 4]), - ]) - - TreeProcessor(steps).run( [NumNote("num", 3)] ) - -def test_copy(): - steps = Copy([ - Sequential([ - SumByConstant(6), - AssertStep([lambda items: items[0].item == 8]) - ]), - Sequential([ - DivByConstant(2), - AssertStep([lambda items: items[0].item == 1]) - ]) - ]) - - TreeProcessor(steps).run( [NumNote("num", 2)] ) - -def test_batchsteps_are_flushed(): - notes = [NumNote("num", 1), NumNote("num", 2), NumNote("num", 3)] - steps = Sequential([ - Print(), - SumByConstant(2, batch_size=2), - ]) - - TreeProcessor(steps).run( notes ) - assert notes[0].items["num"][0].item == 3 - assert notes[1].items["num"][0].item == 4 - assert notes[2].items["num"][0].item == 5 - -def test_batchstep_returns_only_complete_records(): - steps = Sequential([ - SumByConstant(2, batch_size=2), - AssertStep([ - lambda items: items[0].item == 3, - lambda items: items[0].item == 4, - lambda items: items[1].item == 5 - ]), - ]) - - TreeProcessor(steps).run( [NumNote("num", 1), NumListNote("nums", [2, 3])] ) - -def test_example_tree(): - steps = Sequential([ - SumByConstant(5), - Copy([ - Sequential([ - MulByConstant(2), - DivByConstant(4), - AssertStep([lambda items: items[0].item == 5, lambda items: items[0].item == 6]), - ]), - Sequential([ - MulByConstant(4, batch_size=2), - AssertStep([lambda items: items[0].item == 40, lambda items: items[0].item == 48]), - ]) - ]) - ]) - - TreeProcessor(steps).run( LoadJSONL("test_inputs/example_tree_inputs.jsonl") ) - -def test_multiprocessing(): - class MultiprocessingStep(BatchStep): - def __init__(self): - super().__init__(item_key="num", batch_size=2) - MultiprocessingStep.curr_os_pid = os.getpid() # Should be main process pid - - @staticmethod - def load_fn(item) -> any: - # This should be called in a separate process each time - assert MultiprocessingStep.curr_os_pid != os.getpid() - MultiprocessingStep.curr_os_pid = os.getpid() - print("Loading item", item) - return item - - @staticmethod - def dump_fn(items) -> None: - # This should be called in a separate process each time - assert MultiprocessingStep.curr_os_pid != os.getpid() - MultiprocessingStep.curr_os_pid = os.getpid() - print("Dumping batch", items) - assert len(items) == 2, "Items should be batched" - - def exec(self, data: StepData) -> StepOutput: - items, records, completed = data - assert len(items) == 2, "Items should be batched" - assert len(records) == 2, "Records should be batched" - assert len(completed) == 2, "Completed records not in batch" - self.start_dump(items, self.dump_fn) - return { - "_next": completed - } - TreeProcessor(MultiprocessingStep(), num_workers=2).run( [NumNote("num", 1), NumNote("num", 2)] ) diff --git a/tests/test_steps.py b/tests/test_steps.py deleted file mode 100644 index 7b98168..0000000 --- a/tests/test_steps.py +++ /dev/null @@ -1,18 +0,0 @@ -from graphbook.steps.base import Step, Note -import sys -# sys.path.append("src") - -def test_set_child_is_idempotent(): - a = Step("A", None) - b = Step("B", None) - a.set_child(b) - a.set_child(b) - assert len(a.children["out"]) == 1 - -def test_remove_children(): - a = Step("A", None) - b = Step("B", None) - a.set_child(b) - a.remove_children() - assert len(a.children) == 0 - assert len(b.parents) == 0 diff --git a/web/src/components/Flow.tsx b/web/src/components/Flow.tsx index 31bed58..9a759fb 100644 --- a/web/src/components/Flow.tsx +++ b/web/src/components/Flow.tsx @@ -392,6 +392,7 @@ export default function Flow({ filename }) { isValidConnection={isValidConnection} onNodeDragStop={onNodeDragStop} onNodesDelete={onNodesDelete} + preventScrolling={true} > {notificationCtxt} diff --git a/web/src/components/Nodes/Node.tsx b/web/src/components/Nodes/Node.tsx index 1c0c1bc..9bfbd78 100644 --- a/web/src/components/Nodes/Node.tsx +++ b/web/src/components/Nodes/Node.tsx @@ -1,7 +1,7 @@ -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { CSSProperties, useCallback, useEffect, useMemo, useState } from 'react'; import { Handle, Position, useNodes, useEdges, useReactFlow, useOnSelectionChange } from 'reactflow'; -import { Card, Collapse, Badge, Flex, Button, Descriptions, Image, theme } from 'antd'; -import { SearchOutlined, FileTextOutlined, CaretRightOutlined } from '@ant-design/icons'; +import { Card, Collapse, Badge, Flex, Button, Image, Tabs, theme, Space } from 'antd'; +import { SearchOutlined, FileTextOutlined, CaretRightOutlined, FileImageOutlined, CodeOutlined } from '@ant-design/icons'; import { Widget, isWidgetType } from './Widgets'; import { Graph } from '../../graph'; import { useRunState } from '../../hooks/RunState'; @@ -13,11 +13,12 @@ import { useNotification } from '../../hooks/Notification'; import { useSettings } from '../../hooks/Settings'; import { SerializationErrorMessages } from '../Errors'; import type { LogEntry, Parameter } from '../../utils'; +import ReactJson from '@microlink/react-json-view'; const { Panel } = Collapse; const { useToken } = theme; type QuickViewEntry = { - [key: string]: object; + [key: string]: any; }; export function WorkflowStep({ id, data, selected }) { @@ -215,48 +216,102 @@ function Monitor({ quickViewData, logsData }) { function QuickviewCollapse({ data }) { const [settings, _] = useSettings(); + const globalTheme = theme.useToken().theme; + + const tabItems = useCallback((noteData) => { + let data: any = []; + if (settings.quickviewShowNotes) { + data.push({ + key: '0', + label: 'Note', + icon: , + children: ( + + ), + }); + } + if (settings.quickviewShowImages) { + data.push({ + key: '1', + label: 'Images', + icon: , + children: ( + + ), + }); + } + return data; + }, [settings]); + return ( { Object.entries(data).map(([key, value], i) => { + return ( + + + + ); + }) + } + + ); +} - const imageItems = Object.entries(value).filter(([_, itemList]) => { - if (!Array.isArray(itemList)) { - return false; - } - return itemList.filter(item => item.type?.slice(0, 5) === 'image').length > 0; - }).map(([itemKey, itemList]) => { - const images = itemList.filter(item => item.type?.slice(0, 5) === 'image'); - return { - key: itemKey, - label: itemKey, - span: 1, - children: ( - - { - images.map((item, i) => ( - - )) - } - - ) - }; - }); +function EntryImages({ entry, style }: { entry: QuickViewEntry, style: CSSProperties | undefined }) { + const [settings, _] = useSettings(); + + const imageEntries = useMemo(() => { + let entries: any = {}; + Object.entries(entry).forEach(([key, item]) => { + let imageItems: any = []; + if (Array.isArray(item)) { + imageItems = item.filter(item => item.type?.slice(0, 5) === 'image').map(item => item.value); + } else { + if (item.type?.slice(0, 5) === 'image') { + imageItems.push(item.value); + } + } + if (imageItems.length > 0) { + entries[key] = imageItems; + } + }); + return entries; + }, [entry]); + return ( + + { + Object.entries(imageEntries).map(([key, images]) => { return ( - - -
- {JSON.stringify(value, null, 2)} -
+ +
{key}
+ { - imageItems.length > 0 && + images.map((image, i) => ( + + )) } - + + ); }) } - + ); } diff --git a/web/src/components/Nodes/node.css b/web/src/components/Nodes/node.css index c515d7d..a8be49b 100644 --- a/web/src/components/Nodes/node.css +++ b/web/src/components/Nodes/node.css @@ -280,6 +280,10 @@ textarea.code { align-items: center; } +.workflow-node .quickview .ant-tabs .ant-tabs-tab { + font-size: .6rem; +} + .react-flow__node.react-flow__node-group:has(.collapsed) { width: auto !important; height: auto !important; diff --git a/web/src/components/Settings.tsx b/web/src/components/Settings.tsx index d04e406..5bc4ef3 100644 --- a/web/src/components/Settings.tsx +++ b/web/src/components/Settings.tsx @@ -1,5 +1,5 @@ import { useEffect, useState, useCallback } from 'react'; -import { Switch, Input, Typography, Flex, theme, Button, Space } from 'antd'; +import { Switch, Input, Typography, Flex, theme, Button, Space, Checkbox } from 'antd'; import { API } from '../api'; import { useSettings } from '../hooks/Settings'; import React from 'react'; @@ -21,7 +21,6 @@ export default function Settings() { }, []); const setMediaVar = useCallback((name, value) => { - console.log(value); setMediaSettings({ ...mediaSettings, [name]: value }); API.setMediaServerVar(name, value); }, [mediaSettings]); @@ -34,8 +33,16 @@ export default function Settings() { setClientSetting('mediaServerHost', value); }, []); + const setShowNotes = useCallback((event) => { + setClientSetting('quickviewShowNotes', event.target.checked); + }, []); + + const setShowImages = useCallback((event) => { + setClientSetting('quickviewShowImages', event.target.checked); + }, []); + return ( -
+
Client Settings Server Settings setMediaVar('root_path', value)} /> + Quickview Settings + Show Notes + Show Images
); } diff --git a/web/src/hooks/Settings.ts b/web/src/hooks/Settings.ts index 7d4c2ec..20cfb1a 100644 --- a/web/src/hooks/Settings.ts +++ b/web/src/hooks/Settings.ts @@ -7,6 +7,8 @@ let settings = { mediaServerHost: "localhost:8006", monitorDataColumns: MONITOR_DATA_COLUMNS, monitorLogsShouldScrollToBottom: true, + quickviewShowNotes: true, + quickviewShowImages: true, }; const storedSettings = localStorage.getItem('settings'); From 402bbcdad5ebfd2f30231a4842ea829a22bb8296 Mon Sep 17 00:00:00 2001 From: "Richard S. Franklin" Date: Fri, 23 Aug 2024 14:47:29 -0700 Subject: [PATCH 17/21] Worfklow and Node Documentation (#73) --- graphbook/doc2md.py | 32 + graphbook/exports.py | 17 + graphbook/main.py | 39 +- graphbook/steps/arithmetic.py | 2 +- graphbook/steps/io.py | 14 + graphbook/web.py | 45 +- web/package-lock.json | 1118 ++++++++++++++++++- web/package.json | 1 + web/src/api.ts | 13 + web/src/components/Docs.tsx | 179 +++ web/src/components/Flow.tsx | 16 +- web/src/components/LeftPanel/Filesystem.tsx | 42 +- web/src/components/LeftPanel/LeftPanel.tsx | 2 +- web/src/components/LeftPanel/filesystem.css | 17 +- web/src/components/LeftPanel/panel.css | 18 +- web/src/index.css | 2 +- 16 files changed, 1494 insertions(+), 63 deletions(-) create mode 100644 graphbook/doc2md.py create mode 100644 web/src/components/Docs.tsx diff --git a/graphbook/doc2md.py b/graphbook/doc2md.py new file mode 100644 index 0000000..2ec9d92 --- /dev/null +++ b/graphbook/doc2md.py @@ -0,0 +1,32 @@ +from textwrap import dedent + + +def convert_to_md(docstring: str) -> str: + # docstring = docstring.strip() + docstring = dedent(docstring) + lines = docstring.split("\n") + md = "" + bold_key = False + for line in lines: + if line.strip() == "Args:": + md += "**Parameters**" + bold_key = True + elif line.strip() == "Returns:": + md += "**Returns**" + bold_key = False + elif line.strip() == "Raises:": + md += "**Raises**" + bold_key = False + else: + if bold_key: + entry = line.split(":") + if len(entry) > 1: + key = entry[0].strip() + value = entry[1].strip() + md += f"- **{key}**: {value}" + else: + md += line + md += "\n" + md = md.strip() + + return md diff --git a/graphbook/exports.py b/graphbook/exports.py index 65eb154..cd47e27 100644 --- a/graphbook/exports.py +++ b/graphbook/exports.py @@ -1,6 +1,7 @@ import graphbook.steps as steps import graphbook.resources.base as rbase import graphbook.custom_nodes as custom_nodes +from graphbook.doc2md import convert_to_md from aiohttp import web default_exported_steps = { @@ -55,6 +56,22 @@ def get_resources(self): def get_all(self): return {"steps": self.get_steps(), "resources": self.get_resources()} + def get_step_docstring(self, name): + if name in self.exported_steps: + docstring = self.exported_steps[name].__doc__ + if docstring is not None: + docstring = convert_to_md(docstring) + return docstring + return None + + def get_resource_docstring(self, name): + if name in self.exported_resources: + docstring = self.exported_resources[name].__doc__ + if docstring is not None: + docstring = convert_to_md(docstring) + return docstring + return None + def get_exported_nodes(self): # Create directory structure for nodes based on their category def create_dir_structure(nodes): diff --git a/graphbook/main.py b/graphbook/main.py index a0a174c..6d44741 100644 --- a/graphbook/main.py +++ b/graphbook/main.py @@ -1,11 +1,16 @@ #!/usr/bin/env python import argparse from graphbook.web import start_web +import os.path as osp DESCRIPTION = """ Graphbook | ML Workflow Framework """ +workflow_dir = "./workflow" +nodes_dir = "custom_nodes" +docs_dir = "docs" + def get_args(): parser = argparse.ArgumentParser( @@ -22,14 +27,44 @@ def get_args(): parser.add_argument("--graph_port", type=int, default=8005) parser.add_argument("--media_port", type=int, default=8006) parser.add_argument("--web_port", type=int, default=8007) - parser.add_argument("--workflow_dir", type=str, default="./workflow") - parser.add_argument("--nodes_dir", type=str, default="./workflow/custom_nodes") + parser.add_argument( + "--root_dir", + type=str, + help="If setting this directory, workflow_dir, nodes_dir, and docs_dir will be ignored", + ) + parser.add_argument( + "--workflow_dir", + type=str, + default=workflow_dir, + help="Path to the workflow directory", + ) + parser.add_argument( + "--nodes_dir", + type=str, + default=osp.join(workflow_dir, nodes_dir), + help="Path to the custom nodes directory", + ) + parser.add_argument( + "--docs_dir", + type=str, + default=osp.join(workflow_dir, docs_dir), + help="Path to the docs directory", + ) return parser.parse_args() +def resolve_paths(args): + if args.root_dir: + args.workflow_dir = args.root_dir + args.nodes_dir = osp.join(args.root_dir, nodes_dir) + args.docs_dir = osp.join(args.root_dir, docs_dir) + return args + + def main(): args = get_args() + args = resolve_paths(args) start_web(args) diff --git a/graphbook/steps/arithmetic.py b/graphbook/steps/arithmetic.py index d1b4a98..5144c4b 100644 --- a/graphbook/steps/arithmetic.py +++ b/graphbook/steps/arithmetic.py @@ -1,4 +1,4 @@ -from graphbook.steps.base import StepData, StepOutput, Note, any, BatchStep +from graphbook.steps.base import StepData, StepOutput, Note, BatchStep from typing import List diff --git a/graphbook/steps/io.py b/graphbook/steps/io.py index 134dc3c..832d628 100644 --- a/graphbook/steps/io.py +++ b/graphbook/steps/io.py @@ -3,6 +3,13 @@ class LoadJSONL(SourceStep): + """ + Loads input JSONL file and returns a list of Note objects. + + Args: + jsonl_path (str): Path to the JSONL file. + """ + RequiresInput = False Parameters = { "jsonl_path": { @@ -25,6 +32,13 @@ def load(self) -> StepOutput: class DumpJSONL(Step): + """ + Writes Note objects as individual JSONs into a JSONL file. + + Args: + jsonl_path (str): Path to the JSONL file. + """ + RequiresInput = True Parameters = { "jsonl_path": { diff --git a/graphbook/web.py b/graphbook/web.py index 7e2a41c..ea47c60 100644 --- a/graphbook/web.py +++ b/graphbook/web.py @@ -49,10 +49,11 @@ def __init__( processor_pause_event: mp.Event, view_manager_queue: mp.Queue, close_event: mp.Event, + root_path: str, + custom_nodes_path: str, + docs_path: str, host="0.0.0.0", port=8005, - root_path="./workflow", - custom_nodes_path="./workflow/custom_nodes", ): self.host = host self.port = port @@ -195,6 +196,32 @@ async def get_output_note(request: web.Request) -> web.Response: async def get_run_state(request: web.Request) -> web.Response: res = poll_conn_for(state_conn, ProcessorStateRequest.GET_RUNNING_STATE) return web.json_response(res) + + @routes.get(r"/docs/{path:.+}") + async def get_docs(request: web.Request): + path = request.match_info.get("path") + fullpath = osp.join(docs_path, path) + if osp.exists(fullpath): + with open(fullpath, "r") as f: + file_contents = f.read() + d = {"content": file_contents} + return web.json_response(d) + else: + return web.json_response( + {"reason": "/%s: No such file or directory." % fullpath}, status=404 + ) + + @routes.get("/step_docstring/{name}") + async def get_step_docstring(request: web.Request): + name = request.match_info.get("name") + docstring = self.node_hub.get_step_docstring(name) + return web.json_response({"content": docstring}) + + @routes.get("/resource_docstring/{name}") + async def get_resource_docstring(request: web.Request): + name = request.match_info.get("name") + docstring = self.node_hub.get_resource_docstring(name) + return web.json_response({"content": docstring}) @routes.get("/fs") @routes.get(r"/fs/{path:.+}") @@ -383,6 +410,7 @@ def create_graph_server( close_event, root_path, custom_nodes_path, + docs_path, ): server = GraphServer( cmd_queue, @@ -392,6 +420,7 @@ def create_graph_server( close_event, root_path=root_path, custom_nodes_path=custom_nodes_path, + docs_path=docs_path, host=args.host, port=args.graph_port, ) @@ -409,12 +438,15 @@ def start_web(args): view_manager_queue = mp.Queue() close_event = mp.Event() pause_event = mp.Event() - root_path = args.workflow_dir + workflow_dir = args.workflow_dir custom_nodes_path = args.nodes_dir - if not osp.exists(root_path): - os.mkdir(root_path) + docs_path = args.docs_dir + if not osp.exists(workflow_dir): + os.mkdir(workflow_dir) if not osp.exists(custom_nodes_path): os.mkdir(custom_nodes_path) + if not osp.exists(docs_path): + os.mkdir(docs_path) processes = [ mp.Process(target=create_media_server, args=(args,)), mp.Process(target=create_web_server, args=(args,)), @@ -427,8 +459,9 @@ def start_web(args): pause_event, view_manager_queue, close_event, - root_path, + workflow_dir, custom_nodes_path, + docs_path ), ), ] diff --git a/web/package-lock.json b/web/package-lock.json index c406770..06870e5 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,12 +1,12 @@ { "name": "graphbook-web", - "version": "0.4.0", + "version": "0.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "graphbook-web", - "version": "0.4.0", + "version": "0.4.3", "dependencies": { "@codemirror/lang-python": "^6.1.5", "@microlink/react-json-view": "^1.23.1", @@ -19,6 +19,7 @@ "re-resizable": "^6.9.17", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-markdown": "^9.0.1", "reactflow": "^11.11.3", "reaviz": "^15.18.3" }, @@ -1931,28 +1932,62 @@ "@types/d3-selection": "*" } }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", - "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", - "dev": true + "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "dependencies": { + "@types/estree": "*" + } }, "node_modules/@types/geojson": { "version": "7946.0.14", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz", "integrity": "sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==" }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" + }, "node_modules/@types/prop-types": { "version": "15.7.12", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz", - "integrity": "sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==", - "devOptional": true + "integrity": "sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==" }, "node_modules/@types/react": { "version": "18.2.69", "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.69.tgz", "integrity": "sha512-W1HOMUWY/1Yyw0ba5TkCV+oqynRjG7BnteBB+B7JmAK7iw3l2SW+VGOxL+akPweix6jk2NNJtyJKpn4TkpfK3Q==", - "devOptional": true, "dependencies": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -1971,8 +2006,12 @@ "node_modules/@types/scheduler": { "version": "0.16.8", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz", - "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==", - "devOptional": true + "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==" + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" }, "node_modules/@uiw/codemirror-extensions-basic-setup": { "version": "4.21.25", @@ -2068,8 +2107,7 @@ "node_modules/@ungap/structured-clone": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true + "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" }, "node_modules/@upsetjs/venn.js": { "version": "1.4.2", @@ -2416,6 +2454,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -2535,6 +2582,15 @@ } ] }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/center-align": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz", @@ -2561,6 +2617,42 @@ "node": ">=4" } }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/chroma-js": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chroma-js/-/chroma-js-2.4.2.tgz", @@ -2615,6 +2707,15 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/compute-scroll-into-view": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/compute-scroll-into-view/-/compute-scroll-into-view-3.1.0.tgz", @@ -2986,7 +3087,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -3007,6 +3107,18 @@ "node": ">=0.10.0" } }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/deep-equal": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.2.tgz", @@ -3072,6 +3184,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -3613,6 +3745,15 @@ "node": ">=4.0" } }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -3622,6 +3763,11 @@ "node": ">=0.10.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4128,11 +4274,58 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.0.tgz", + "integrity": "sha512-H/y0+IWPdsLLS738P8tDnrQ8Z+dj12zQQ6WC11TIM21C8WFVoIxcqWXf2H3hiTVZjF1AWqoimGwrTWecWrnmRQ==", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/highlight-words-core": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/highlight-words-core/-/highlight-words-core-1.2.2.tgz", "integrity": "sha512-BXUKIkUuh6cmmxzi5OIbUJxrG8OAk2MqoL1DtO3Wo9D2faJg2ph5ntyuQeLqaHJmzER6H5tllCDA9ZnNe9BVGg==" }, + "node_modules/html-url-attributes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.0.tgz", + "integrity": "sha512-/sXbVCWayk6GDVg3ctOX6nxaVj7So40FcFAnWlWGNAB1LpYKcV5Cd10APjPjW80O7zYW2MsjBV4zZ7IZO5fVow==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/human-format": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/human-format/-/human-format-1.2.0.tgz", @@ -4189,6 +4382,11 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/inline-style-parser": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.3.tgz", + "integrity": "sha512-qlD8YNDqyTKTyuITrDOffsl6Tdhv+UC4hcdAVuQsK4IMQ99nSgd1MIA/Q+jQYoh9r3hVUXhYh7urSRmXPkW04g==" + }, "node_modules/internal-slot": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz", @@ -4215,6 +4413,28 @@ "resolved": "https://registry.npmjs.org/invert-color/-/invert-color-2.0.0.tgz", "integrity": "sha512-9s6IATlhOAr0/0MPUpLdMpk81ixIu8IqwPwORssXBauFT/4ff/iyEOcojd0UYuPwkDbJvL1+blIZGhqVIaAm5Q==" }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-arguments": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", @@ -4341,6 +4561,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -4389,6 +4618,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/is-map": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz", @@ -4435,6 +4673,17 @@ "node": ">=8" } }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-regex": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz", @@ -4759,6 +5008,15 @@ "node": ">=0.10.0" } }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -4779,6 +5037,151 @@ "yallist": "^3.0.2" } }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.1.tgz", + "integrity": "sha512-aJEUyzZ6TzlsX2s5B4Of7lN7EQtAxvtradMMglCQDyaTFgse6CmtmdJ15ElnVRlCg1vpNyVtbem0PWzlNieZsA==", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.0.tgz", + "integrity": "sha512-fGCu8eWdKUKNu5mohVGkhBXCXGnOTLuFqOvGMvdikr+J1w7lDJgxThOKpwRWzzbyXAU2hhSwsmssOY4yTokluw==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.1.2.tgz", + "integrity": "sha512-eKMQDeywY2wlHc97k5eD8VC+9ASMjN8ItEZQNGwJ6E0XWKiW/Z0V5/H8pvoXUf+y+Mj0VIgeRRbujBmFn4FTyA==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-remove-position": "^5.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", + "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.0.tgz", + "integrity": "sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ==", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/memoize-bind": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/memoize-bind/-/memoize-bind-1.0.3.tgz", @@ -4797,6 +5200,427 @@ "resolved": "https://registry.npmjs.org/memoize-weak/-/memoize-weak-1.0.2.tgz", "integrity": "sha512-gj39xkrjEw7nCn4nJ1M5ms6+MyMlyiGmttzsqAUsAKn6bYKwuTHh/AO3cKPF8IBrTIYTxb0wWXFs3E//Y8VoWQ==" }, + "node_modules/micromark": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.0.tgz", + "integrity": "sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.1.tgz", + "integrity": "sha512-CUQyKr1e///ZODyD1U3xit6zXwy1a8q2a1S1HKtIlmgvurrEpaw/Y9y6KSIbF8P59cn/NjzHyO+Q2fAyYLQrAA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.0.tgz", + "integrity": "sha512-j9DGrQLm/Uhl2tCzcbLhy5kXsgkHUrjJHg4fFAeoMRwJmJerT9aw4FEhIbZStWN8A3qMwOp1uzHr4UL8AInxtA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.0.tgz", + "integrity": "sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.0.tgz", + "integrity": "sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.0.tgz", + "integrity": "sha512-jY8CSxmpWLOxS+t8W+FG3Xigc0RDQA9bKMY/EwILvsesiRniiVMejYTE4wumNc2f4UbAa4WsHqe3J1QS1sli+A==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.0.tgz", + "integrity": "sha512-28kbwaBjc5yAI1XadbdPYHX/eDnqaUFVikLwrO7FDnKG7lpgxnvk/XGRhX/PN0mOZ+dBSZ+LgunHS+6tYQAzhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.0.tgz", + "integrity": "sha512-KvOVV+X1yLBfs9dCBSopq/+G1PcgT3lAK07mC4BzXi5E7ahzMAF8oIupDDJ6mievI6F+lAATkbQQlQixJfT3aQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.0.tgz", + "integrity": "sha512-anK8SWmNphkXdaKgz5hJvGa7l00qmcaUQoMYsBwDlSKFKjc6gjGXPDw3FNL3Nbwq5L8gE+RCbGqTw49FK5Qyvg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.0.tgz", + "integrity": "sha512-S0ze2R9GH+fu41FA7pbSqNWObo/kzwf8rN/+IGlW/4tC6oACOs8B++bh+i9bVyNnwCcuksbFwsBme5OCKXCwIw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.0.tgz", + "integrity": "sha512-vZZio48k7ON0fVS3CUgFatWHoKbbLTK/rT7pzpJ4Bjp5JjkZeasRfrS9wsBdDJK2cJLHMckXZdzPSSr1B8a4oQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.1.tgz", + "integrity": "sha512-bmkNc7z8Wn6kgjZmVHOX3SowGmVdhYS7yBpMnuMnPzDq/6xwVA604DuOXMZTO1lvq01g+Adfa0pE2UKGlxL1XQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.0.tgz", + "integrity": "sha512-r4Sc6leeUTn3P6gk20aFMj2ntPwn6qpDZqWvYmAG6NgvFTIlj4WtrAudLi65qYoaGdXYViXYw2pkmn7QnIFasA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.0.tgz", + "integrity": "sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.0.tgz", + "integrity": "sha512-xNn4Pqkj2puRhKdKTm8t1YHC/BAjx6CEwRFXntTaRf/x16aqka6ouVoutm+QdkISTlT7e2zU7U4ZdlDLJd2Mcw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.0.tgz", + "integrity": "sha512-2xhYT0sfo85FMrUPtHcPo2rrp1lwbDEEzpx7jiH2xXJLqBuy4H0GgXk5ToU8IEwoROtXuL8ND0ttVa4rNqYK3w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.0.tgz", + "integrity": "sha512-6KU6qO7DZ7GJkaCgwBNtplXCvGkJToU86ybBAUdavvgsCiG8lSSvYxr9MhwmQ+udpzywHsl4RpGJsYWG1pDOcA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.0.tgz", + "integrity": "sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.1.tgz", + "integrity": "sha512-jZNtiFl/1aY73yS3UGQkutD0UbhTt68qnRpw2Pifmz5wV9h8gOVsN70v+Lq/f1rKaU/W8pxRe8y8Q9FX1AOe1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.0.tgz", + "integrity": "sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, + "node_modules/micromark-util-types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.0.tgz", + "integrity": "sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ] + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -4838,8 +5662,7 @@ "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/name-initials": { "version": "0.1.3", @@ -5084,6 +5907,30 @@ "node": ">=6" } }, + "node_modules/parse-entities": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.1.tgz", + "integrity": "sha512-SWzvYcSJh4d/SGLIOQfZ/CoNv6BTlI6YEQ7Nj82oDVnRpwe/Z/F1EMx42x3JAOwGBlCjeCH0BRJQbQ/opHL17w==", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -5192,6 +6039,15 @@ "react-is": "^16.13.1" } }, + "node_modules/property-information": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.5.0.tgz", + "integrity": "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -5935,6 +6791,31 @@ "resolved": "https://registry.npmjs.org/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz", "integrity": "sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA==" }, + "node_modules/react-markdown": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.1.tgz", + "integrity": "sha512-186Gw/vF1uRkydbsOIkcGXw7aHq0sZOCRFFjGrr7b9+nVZg4UfA4enXCaxm4fUzecU38sWfrNDitGhshuU7rdg==", + "dependencies": { + "@types/hast": "^3.0.0", + "devlop": "^1.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "html-url-attributes": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "unified": "^11.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=18", + "react": ">=18" + } + }, "node_modules/react-refresh": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.0.tgz", @@ -6084,6 +6965,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.0.tgz", + "integrity": "sha512-z3tJrAs2kIs1AqIIy6pzHmAHlF1hWQ+OdY4/hv+Wxe35EhyLKcajL33iUEn3ScxtFox9nUvRufR/Zre8Q08H/g==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", @@ -6391,6 +7303,15 @@ "node": ">=0.8.0" } }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/string-convert": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", @@ -6468,6 +7389,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -6497,6 +7431,14 @@ "resolved": "https://registry.npmjs.org/style-mod/-/style-mod-4.1.2.tgz", "integrity": "sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==" }, + "node_modules/style-to-object": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.6.tgz", + "integrity": "sha512-khxq+Qm3xEyZfKd/y9L3oIWQimxuc4STrQKtQn8aSDRHb8mFgpukgX1hdzfrMEW6JCjyJ8p89x+IUMVnCBI1PA==", + "dependencies": { + "inline-style-parser": "0.2.3" + } + }, "node_modules/stylis": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.3.1.tgz", @@ -6633,6 +7575,24 @@ "url": "https://github.com/sponsors/chrvadala" } }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/tslib": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", @@ -6791,6 +7751,100 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-remove-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-5.0.0.tgz", + "integrity": "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -6875,6 +7929,33 @@ "react": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/vfile": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.2.tgz", + "integrity": "sha512-zND7NlS8rJYb/sPqkb13ZvbbUoExdbi4w3SfRrMq6R3FvnLQmmfpajJNITuuYm6AZ5uao9vy4BAos3EXBPf2rg==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/vite": { "version": "5.2.6", "resolved": "https://registry.npmjs.org/vite/-/vite-5.2.6.tgz", @@ -7117,6 +8198,15 @@ "optional": true } } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } } } } diff --git a/web/package.json b/web/package.json index 960ecb4..b677fe4 100644 --- a/web/package.json +++ b/web/package.json @@ -21,6 +21,7 @@ "re-resizable": "^6.9.17", "react": "^18.2.0", "react-dom": "^18.2.0", + "react-markdown": "^9.0.1", "reactflow": "^11.11.3", "reaviz": "^15.18.3" }, diff --git a/web/src/api.ts b/web/src/api.ts index 606d80e..bee95c9 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -287,6 +287,19 @@ export class ServerAPI { return null; } + public async getWorkflowDoc(workflowFilename: string) { + const filename = workflowFilename.replace('.json', '.md'); + return await this.get(`docs/${filename}`); + } + + public async getStepDocstring(name: string) { + return await this.get(`step_docstring/${name}`); + } + + public async getResourceDocstring(name: string) { + return await this.get(`resource_docstring/${name}`); + } + /** * Graph API */ diff --git a/web/src/components/Docs.tsx b/web/src/components/Docs.tsx new file mode 100644 index 0000000..718a85e --- /dev/null +++ b/web/src/components/Docs.tsx @@ -0,0 +1,179 @@ +import React, { useMemo, useState, useEffect, useCallback, useLayoutEffect } from "react"; +import { Divider, Flex, Typography, Collapse, theme } from "antd"; +import { MenuFoldOutlined, MenuUnfoldOutlined } from "@ant-design/icons"; +import { useFilename } from "../hooks/Filename"; +import { useAPI } from "../hooks/API"; +import { useNodes } from "reactflow"; +import type { CollapseProps } from "antd/lib/collapse"; +import Markdown from 'react-markdown'; +const { useToken } = theme; +const { Text } = Typography; + +type NodeDoc = { + name: string; + content: string; +}; + +const DefaultDoc = + ` +### No documentation found. + +To add documentation place a markdown file with the same name as the workflow file (with the .md extension) inside your specified docs directory. +By default, this is located in your \`docs/\` directory. + +`; + +const getExampleStr = (name: string) => { + return `For example, make a file named \`docs/${name}.md\` and add your documentation there.`; +}; + +export function Docs() { + const { token } = useToken(); + const [hidden, setHidden] = useState(false); + const [nodeDocs, setNodeDocs] = useState([]); + const [windowHeight, setWindowHeight] = useState(0); + const nodes = useNodes(); + const API = useAPI(); + const filename = useFilename(); + const initialDocStr = useMemo(() => DefaultDoc + getExampleStr(filename.split('.')[0]), [filename]); + const [workflowDoc, setWorkflowDoc] = useState(initialDocStr); + + useEffect(() => { + const loadDocs = async () => { + if (!API || !filename) { + return; + } + const docs = await API.getWorkflowDoc(filename); + if (docs?.content) { + setWorkflowDoc(docs.content); + } + }; + + loadDocs(); + }, [API, filename]); + + useLayoutEffect(() => { + const updateWindowHeight = () => { + setWindowHeight(window.innerHeight); + }; + + window.addEventListener('resize', updateWindowHeight); + updateWindowHeight(); + return () => window.removeEventListener('resize', updateWindowHeight); + }, []); + + useEffect(() => { + if (!nodes || !API) { + return; + } + + const loadNodeDocs = async () => { + const n = nodes as any[]; + const uniqueNodes = new Set(n.filter(n => n.type === 'step' || n.type === 'resource').map(n => n.data.name)); + const nodeMap = {}; + n.forEach(node => { nodeMap[node.data.name] = node }); + + const docs = await Promise.all([...uniqueNodes].map(async (nodeName) => { + const node = nodeMap[nodeName]; + const doc = node.type === 'step' ? await API.getStepDocstring(nodeName) : await API.getResourceDocstring(nodeName); + return { + name: node.data.name, + content: doc?.content || '(No docstring)' + }; + })); + setNodeDocs(docs); + }; + + loadNodeDocs(); + }, [nodes, API]); + + const containerStyle: React.CSSProperties = useMemo(() => ({ + padding: '5px 10px', + backgroundColor: token.colorBgBase, + borderRadius: token.borderRadius, + color: token.colorTextBase, + height: windowHeight - 80, + width: '400px', + }), [token, windowHeight]); + + const unfoldStyle: React.CSSProperties = useMemo(() => ({ + padding: '10px 5px', + }), []); + + const docSectionStyle: React.CSSProperties = useMemo(() => ({ + flex: 1, + height: 0, + overflow: 'auto', + border: `1px solid ${token.colorBorder}`, + borderRadius: token.borderRadius, + }), [token]); + + const items: CollapseProps['items'] = useMemo(() => { + return nodeDocs.map((doc, i) => { + return { + key: i.toString(), + label: doc.name, + children: {doc.content}, + } + }); + }, [nodeDocs]); + + if (hidden) { + return ( + +
setHidden(false)} > + +
+ + ); + } + + return ( + + + setHidden(true)} /> + {filename} + + + { + workflowDoc && +
+ {workflowDoc} +
+ } + { + nodeDocs.length > 0 && + + Included Nodes + + + } + + ); +} + +function Clickable({ children }) { + const [hovered, setHovered] = useState(false); + const { token } = useToken(); + + const onMouseEnter = useCallback(() => { + setHovered(true); + }, []); + + const onMouseLeave = useCallback(() => { + setHovered(false); + }, []); + + const style = useMemo(() => ({ + backgroundColor: hovered ? token.colorBgLayout : token.colorBgBase, + transition: 'background-color', + transitionDuration: '0.2s', + cursor: 'pointer', + }), [hovered]); + + return ( +
+ {children} +
+ ); +} diff --git a/web/src/components/Flow.tsx b/web/src/components/Flow.tsx index 9a759fb..fd6ef1e 100644 --- a/web/src/components/Flow.tsx +++ b/web/src/components/Flow.tsx @@ -8,7 +8,7 @@ import ReactFlow, { useNodes, useEdges } from 'reactflow'; -import { Button, Flex, theme } from 'antd'; +import { Button, Flex, Space, theme } from 'antd'; import { ClearOutlined, CaretRightOutlined, PauseOutlined } from '@ant-design/icons'; import { Graph } from '../graph.ts'; import AddNode from './AddNode.tsx'; @@ -29,6 +29,7 @@ import { SerializationErrorMessages } from './Errors.tsx'; import { useFilename } from '../hooks/Filename.ts'; import { ReactFlowInstance, Node, Edge, BackgroundVariant } from 'reactflow'; import { ActiveOverlay } from './ActiveOverlay.tsx'; +import { Docs } from './Docs.tsx'; const { useToken } = theme; const makeDroppable = (e) => e.preventDefault(); @@ -395,9 +396,14 @@ export default function Flow({ filename }) { preventScrolling={true} > {notificationCtxt} - - - + +
+
+ +
+ +
+ @@ -458,7 +464,7 @@ function ControlRow() { return (
- +
- +
+ +
{contextMenu && ( )} -
+ - ); - } function DirItem({ title, filename, isRenaming, onRename }) { diff --git a/web/src/components/LeftPanel/LeftPanel.tsx b/web/src/components/LeftPanel/LeftPanel.tsx index cf2ec82..0c4742a 100644 --- a/web/src/components/LeftPanel/LeftPanel.tsx +++ b/web/src/components/LeftPanel/LeftPanel.tsx @@ -25,7 +25,7 @@ export default function LeftPanel({ setWorkflow, onBeginEdit }) { return ( diff --git a/web/src/components/LeftPanel/filesystem.css b/web/src/components/LeftPanel/filesystem.css index c0aa0fb..7957c91 100644 --- a/web/src/components/LeftPanel/filesystem.css +++ b/web/src/components/LeftPanel/filesystem.css @@ -1,9 +1,14 @@ -.filesystem .ant-tree-node-content-wrapper{ +.filesystem { + height: 100%; +} + +.filesystem .ant-tree-node-content-wrapper { display: flex; flex-direction: row; text-overflow: ellipsis; white-space: nowrap; width: 100%; + height: 100%; } .filesystem .ant-tree-title .file-item { @@ -14,6 +19,10 @@ display: block; } +.filesystem .ant-tree-title { + overflow: hidden; +} + .filesystem .fs-icon { margin: 0 2px; padding: 1px; @@ -25,4 +34,8 @@ width: auto; padding: 1px 4px; border-radius: 0%; -} \ No newline at end of file +} + +.filesystem .ant-tree-node-content-wrapper.ant-tree-node-content-wrapper-normal { + overflow: hidden; +} diff --git a/web/src/components/LeftPanel/panel.css b/web/src/components/LeftPanel/panel.css index dea3220..2d6062a 100644 --- a/web/src/components/LeftPanel/panel.css +++ b/web/src/components/LeftPanel/panel.css @@ -1,14 +1,12 @@ -.left-panel { - position: absolute; - left:0; +.ant-layout-sider-children { + display: flex; + flex-direction: column; } -/* .ant-tree-title { - width: 100%; - display: inline-block; +.ant-tabs-content.ant-tabs-content-top { + height: 100%; } -.ant-tree-title .file-item { - width: 100%; - display: inline-block; -} */ \ No newline at end of file +.ant-tabs-tabpane.ant-tabs-tabpane-active { + height: 100%; +} diff --git a/web/src/index.css b/web/src/index.css index 5c34225..c6363af 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -18,4 +18,4 @@ html,body{ width:100vw; margin: 0; padding: 0; -} \ No newline at end of file +} From c380ffb85802a5cbecf271105dc54d427aad99d3 Mon Sep 17 00:00:00 2001 From: Richard Franklin Date: Fri, 23 Aug 2024 16:42:50 -0700 Subject: [PATCH 18/21] Bringing web serving onto Graph server --- graphbook/web.py | 69 ++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/graphbook/web.py b/graphbook/web.py index ea47c60..f64aa60 100644 --- a/graphbook/web.py +++ b/graphbook/web.py @@ -7,8 +7,6 @@ import os.path as osp import re import signal -import http.server -import socketserver import multiprocessing as mp import multiprocessing.connection as mpc import asyncio @@ -17,11 +15,14 @@ from graphbook.state import UIState from graphbook.media import create_media_server from graphbook.utils import poll_conn_for, ProcessorStateRequest + try: import magic except ImportError: magic = None - print("Warn: Optional libmagic library not found. Filesystem will not be able to determine MIME types.") + print( + "Warn: Optional libmagic library not found. Filesystem will not be able to determine MIME types." + ) @web.middleware @@ -52,6 +53,7 @@ def __init__( root_path: str, custom_nodes_path: str, docs_path: str, + web_dir: str | None = None, host="0.0.0.0", port=8005, ): @@ -69,6 +71,15 @@ def __init__( self.app = web.Application( client_max_size=max_upload_size, middlewares=middlewares ) + + self.web_dir = web_dir + if self.web_dir is None: + self.web_dir = osp.join(osp.dirname(__file__), "web") + if not osp.isdir(self.web_dir): + print( + f"Couldn't find web files inside {self.web_dir}. Will not serve web files." + ) + self.web_dir = None @routes.get("/ws") async def websocket_handler(request): @@ -97,7 +108,9 @@ async def websocket_handler(request): @routes.get("/") async def get(request: web.Request) -> web.Response: - return web.Response(text="Ok") + if self.web_dir is None: + return web.HTTPNotFound("No web files found.") + return web.FileResponse(osp.join(self.web_dir, "index.html")) @routes.post("/run") async def run_all(request: web.Request) -> web.Response: @@ -191,12 +204,12 @@ async def get_output_note(request: web.Request) -> web.Response: return web.json_response(res) return web.json_response({"error": "Could not get output note."}) - + @routes.get("/state") async def get_run_state(request: web.Request) -> web.Response: res = poll_conn_for(state_conn, ProcessorStateRequest.GET_RUNNING_STATE) return web.json_response(res) - + @routes.get(r"/docs/{path:.+}") async def get_docs(request: web.Request): path = request.match_info.get("path") @@ -210,13 +223,13 @@ async def get_docs(request: web.Request): return web.json_response( {"reason": "/%s: No such file or directory." % fullpath}, status=404 ) - + @routes.get("/step_docstring/{name}") async def get_step_docstring(request: web.Request): name = request.match_info.get("name") docstring = self.node_hub.get_step_docstring(name) return web.json_response({"content": docstring}) - + @routes.get("/resource_docstring/{name}") async def get_resource_docstring(request: web.Request): name = request.match_info.get("name") @@ -368,6 +381,10 @@ async def _async_start(self): def start(self): self.app.router.add_routes(self.routes) + if self.web_dir is not None: + self.app.router.add_routes([ + web.static('/', self.web_dir), + ]) print(f"Starting graph server at {self.host}:{self.port}") self.node_hub.start() try: @@ -376,31 +393,6 @@ def start(self): self.node_hub.stop() print("Exiting graph server") - -class WebServer: - def __init__(self, host, port, web_dir): - self.host = host - self.port = port - if web_dir is None: - web_dir = osp.join(osp.dirname(__file__), "web") - self.cwd = web_dir - self.server = http.server.SimpleHTTPRequestHandler - - def start(self): - if not osp.isdir(self.cwd): - print( - f"Couldn't find web files inside {self.cwd}. Will not start web server." - ) - return - os.chdir(self.cwd) - with socketserver.TCPServer((self.host, self.port), self.server) as httpd: - print(f"Starting web server at {self.host}:{self.port}") - print(f"Visit http://127.0.0.1:{self.port}") - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("Exiting web server") - def create_graph_server( args, cmd_queue, @@ -411,6 +403,7 @@ def create_graph_server( root_path, custom_nodes_path, docs_path, + web_dir, ): server = GraphServer( cmd_queue, @@ -421,17 +414,13 @@ def create_graph_server( root_path=root_path, custom_nodes_path=custom_nodes_path, docs_path=docs_path, + web_dir=web_dir, host=args.host, port=args.graph_port, ) server.start() -def create_web_server(args): - server = WebServer(host=args.host, port=args.web_port, web_dir=args.web_dir) - server.start() - - def start_web(args): cmd_queue = mp.Queue() parent_conn, child_conn = mp.Pipe() @@ -449,7 +438,6 @@ def start_web(args): os.mkdir(docs_path) processes = [ mp.Process(target=create_media_server, args=(args,)), - mp.Process(target=create_web_server, args=(args,)), mp.Process( target=create_graph_server, args=( @@ -461,7 +449,8 @@ def start_web(args): close_event, workflow_dir, custom_nodes_path, - docs_path + docs_path, + args.web_dir, ), ), ] From a2046374e19ae28a2907a1dfd4ee365e1ba08c92 Mon Sep 17 00:00:00 2001 From: Richard Franklin Date: Fri, 23 Aug 2024 19:29:19 -0700 Subject: [PATCH 19/21] Consolidating ports --- Dockerfile | 2 +- README.md | 6 +-- docs/installing.rst | 6 +-- graphbook/main.py | 4 +- graphbook/media.py | 21 ++++------- graphbook/web.py | 25 +++++++++---- web/src/components/Monitor.tsx | 2 +- web/src/components/Nodes/Node.tsx | 6 +-- web/src/components/Settings.tsx | 61 ++++++++++++++++++++++--------- web/src/hooks/Settings.ts | 2 + web/src/utils.ts | 11 +++++- 11 files changed, 94 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9351ee4..3ee41d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,6 @@ RUN poetry install --no-root --no-directory COPY . . RUN make web -EXPOSE 8005 8006 8007 +EXPOSE 8005 8006 CMD ["python", "graphbook/main.py", "--web", "--web_dir", "web/dist"] diff --git a/README.md b/README.md index ca04225..5d63120 100644 --- a/README.md +++ b/README.md @@ -58,14 +58,14 @@ Graphbook is in a very early stage of development, so expect minor bugs and rapi ### Install from PyPI 1. `pip install graphbook` 1. `graphbook` -1. Visit http://localhost:8007 +1. Visit http://localhost:8005 ### Install with Docker 1. Pull and run the downloaded image ```bash - docker run --rm -p 8005:8005 -p 8006:8006 -p 8007:8007 -v $PWD/workflows:/app/workflows rsamf/graphbook:latest + docker run --rm -p 8005:8005 -v $PWD/workflows:/app/workflows rsamf/graphbook:latest ``` -1. Visit http://localhost:8007 +1. Visit http://localhost:8005 Visit the [docs](https://docs.graphbook.ai) to learn more on how to create custom nodes and workflows with Graphbook. diff --git a/docs/installing.rst b/docs/installing.rst index 45221d4..b91155d 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -10,7 +10,7 @@ Install from PyPI #. ``pip install graphbook`` #. ``graphbook`` -#. Visit http://localhost:8007 +#. Visit http://localhost:8005 Install with Docker =================== @@ -19,9 +19,9 @@ Install with Docker .. code-block:: bash - docker run --rm -p 8005:8005 -p 8006:8006 -p 8007:8007 -v $PWD/workflows:/app/workflows rsamf/graphbook:latest + docker run --rm -p 8005:8005 -v $PWD/workflows:/app/workflows rsamf/graphbook:latest -#. Visit http://localhost:8007 +#. Visit http://localhost:8005 Install from Source diff --git a/graphbook/main.py b/graphbook/main.py index 6d44741..94ac24e 100644 --- a/graphbook/main.py +++ b/graphbook/main.py @@ -24,9 +24,9 @@ def get_args(): parser.add_argument("--media_dir", type=str, default="/") parser.add_argument("--web_dir", type=str) parser.add_argument("--host", type=str, default="0.0.0.0") - parser.add_argument("--graph_port", type=int, default=8005) + parser.add_argument("--port", type=int, default=8005) + parser.add_argument("--start_media_server", action="store_true") parser.add_argument("--media_port", type=int, default=8006) - parser.add_argument("--web_port", type=int, default=8007) parser.add_argument( "--root_dir", type=str, diff --git a/graphbook/media.py b/graphbook/media.py index 3401f5d..8d96f1c 100644 --- a/graphbook/media.py +++ b/graphbook/media.py @@ -2,6 +2,7 @@ from aiohttp import web import os.path as osp + @web.middleware async def cors_middleware(request: web.Request, handler): if request.method == "OPTIONS": @@ -18,6 +19,7 @@ async def cors_middleware(request: web.Request, handler): response.headers["Access-Control-Allow-Credentials"] = "true" return response + class MediaServer: def __init__( self, @@ -31,10 +33,8 @@ def __init__( routes = web.RouteTableDef() self.routes = routes middlewares = [cors_middleware] - self.app = web.Application( - middlewares=middlewares - ) - + self.app = web.Application(middlewares=middlewares) + @routes.put("/set") async def set_var_handler(request: web.Request): data = await request.json() @@ -42,18 +42,15 @@ async def set_var_handler(request: web.Request): if root_path: self.root_path = root_path return web.json_response({"root_path": self.root_path}) - + @routes.get(r"/{path:.*}") async def handle(request: web.Request) -> web.Response: path = request.match_info["path"] full_path = osp.join(self.root_path, path) if not osp.exists(full_path): return web.HTTPNotFound() - with open(full_path, "rb") as f: - content = f.read() - return web.Response(body=content) - - + return web.FileResponse(full_path) + async def _async_start(self): runner = web.AppRunner(self.app) await runner.setup() @@ -71,7 +68,5 @@ def start(self): def create_media_server(args): - server = MediaServer( - host=args.host, port=args.media_port, root_path=args.media_dir - ) + server = MediaServer(host=args.host, port=args.media_port, root_path=args.media_dir) server.start() diff --git a/graphbook/web.py b/graphbook/web.py index f64aa60..972f65b 100644 --- a/graphbook/web.py +++ b/graphbook/web.py @@ -71,7 +71,7 @@ def __init__( self.app = web.Application( client_max_size=max_upload_size, middlewares=middlewares ) - + self.web_dir = web_dir if self.web_dir is None: self.web_dir = osp.join(osp.dirname(__file__), "web") @@ -109,9 +109,16 @@ async def websocket_handler(request): @routes.get("/") async def get(request: web.Request) -> web.Response: if self.web_dir is None: - return web.HTTPNotFound("No web files found.") + return web.HTTPNotFound(body="No web files found.") return web.FileResponse(osp.join(self.web_dir, "index.html")) + @routes.get("/media") + async def get_media(request: web.Request) -> web.Response: + path = request.query.get("path", "") + if not osp.exists(path): + return web.HTTPNotFound() + return web.FileResponse(path) + @routes.post("/run") async def run_all(request: web.Request) -> web.Response: data = await request.json() @@ -382,9 +389,11 @@ async def _async_start(self): def start(self): self.app.router.add_routes(self.routes) if self.web_dir is not None: - self.app.router.add_routes([ - web.static('/', self.web_dir), - ]) + self.app.router.add_routes( + [ + web.static("/", self.web_dir), + ] + ) print(f"Starting graph server at {self.host}:{self.port}") self.node_hub.start() try: @@ -393,6 +402,7 @@ def start(self): self.node_hub.stop() print("Exiting graph server") + def create_graph_server( args, cmd_queue, @@ -416,7 +426,7 @@ def create_graph_server( docs_path=docs_path, web_dir=web_dir, host=args.host, - port=args.graph_port, + port=args.port, ) server.start() @@ -437,7 +447,6 @@ def start_web(args): if not osp.exists(docs_path): os.mkdir(docs_path) processes = [ - mp.Process(target=create_media_server, args=(args,)), mp.Process( target=create_graph_server, args=( @@ -454,6 +463,8 @@ def start_web(args): ), ), ] + if args.start_media_server: + processes.append(mp.Process(target=create_media_server, args=(args,))) for p in processes: p.daemon = True diff --git a/web/src/components/Monitor.tsx b/web/src/components/Monitor.tsx index 0e9f9ce..4abc038 100644 --- a/web/src/components/Monitor.tsx +++ b/web/src/components/Monitor.tsx @@ -437,7 +437,7 @@ function NotesView({ stepId, numNotes, type }: NotesViewProps) { { paths.map((path, i) => { return ( - + ); }) } diff --git a/web/src/components/Nodes/Node.tsx b/web/src/components/Nodes/Node.tsx index 9bfbd78..61aef19 100644 --- a/web/src/components/Nodes/Node.tsx +++ b/web/src/components/Nodes/Node.tsx @@ -298,12 +298,12 @@ function EntryImages({ entry, style }: { entry: QuickViewEntry, style: CSSProper { Object.entries(imageEntries).map(([key, images]) => { return ( - +
{key}
- + { images.map((image, i) => ( - + )) } diff --git a/web/src/components/Settings.tsx b/web/src/components/Settings.tsx index 5bc4ef3..9905b1a 100644 --- a/web/src/components/Settings.tsx +++ b/web/src/components/Settings.tsx @@ -1,5 +1,5 @@ import { useEffect, useState, useCallback } from 'react'; -import { Switch, Input, Typography, Flex, theme, Button, Space, Checkbox } from 'antd'; +import { Switch, Input, Typography, Flex, theme, Button, Space, Checkbox, Slider } from 'antd'; import { API } from '../api'; import { useSettings } from '../hooks/Settings'; import React from 'react'; @@ -29,6 +29,10 @@ export default function Settings() { setClientSetting('graphServerHost', value); }, []); + const setUseExternalMediaServer = useCallback((event) => { + setClientSetting('useExternalMediaServer', event.target.checked); + }, []); + const setMediaServerHost = useCallback((value) => { setClientSetting('mediaServerHost', value); }, []); @@ -41,29 +45,50 @@ export default function Settings() { setClientSetting('quickviewShowImages', event.target.checked); }, []); + + const setImageHeight = useCallback((value) => { + setClientSetting('quickviewImageHeight', value); + }, []); + + return (
- Client Settings - { setClientSetting('theme', checked ? "Dark" : "Light") }} - /> - - - Server Settings - setMediaVar('root_path', value)} /> - Quickview Settings - Show Notes - Show Images + + Client Settings + { setClientSetting('theme', checked ? "Dark" : "Light") }} + /> + + + Media Server Settings + Use External Media Server + + setMediaVar('root_path', value)} disabled={!clientSettings.useExternalMediaServer} /> + + Quickview Settings + Show Notes + Show Images + + Image Height + + +
); } function SettingsEntryInput({ name, value, ...optionalProps }) { - const { onChange, onApply, addonBefore } = optionalProps; + const { onChange, onApply, addonBefore, disabled } = optionalProps; const [inputValue, setInputValue] = useState(value); const onChangeSetting = useCallback((value) => { @@ -80,7 +105,7 @@ function SettingsEntryInput({ name, value, ...optionalProps }) { {name} - onChangeSetting(e.target.value)} addonBefore={addonBefore} onPressEnter={onPressEnter} /> + onChangeSetting(e.target.value)} addonBefore={addonBefore} onPressEnter={onPressEnter} disabled={disabled} /> {onApply && } diff --git a/web/src/hooks/Settings.ts b/web/src/hooks/Settings.ts index 20cfb1a..a6cedbd 100644 --- a/web/src/hooks/Settings.ts +++ b/web/src/hooks/Settings.ts @@ -5,10 +5,12 @@ let settings = { theme: "Light", graphServerHost: "localhost:8005", mediaServerHost: "localhost:8006", + useExternalMediaServer: false, monitorDataColumns: MONITOR_DATA_COLUMNS, monitorLogsShouldScrollToBottom: true, quickviewShowNotes: true, quickviewShowImages: true, + quickviewImageHeight: 100, }; const storedSettings = localStorage.getItem('settings'); diff --git a/web/src/utils.ts b/web/src/utils.ts index 98412a2..fb70612 100644 --- a/web/src/utils.ts +++ b/web/src/utils.ts @@ -24,7 +24,16 @@ export const keyRecursively = (obj: Array, childrenKey: string = "children" return keyRec(obj); } -export const getMediaPath = (mediaHost: string, path: string): string => { +export const getMediaPath = (settings: any, path: string): string => { + if (!settings.useExternalMediaServer) { + let graphHost = settings.graphServerHost; + if (!graphHost.startsWith('http')) { + graphHost = 'http://' + graphHost; + } + return `${graphHost}/media?path=${path}`; + } + + let mediaHost = settings.mediaServerHost; if (!mediaHost.startsWith('http')) { mediaHost = 'http://' + mediaHost; } From 7c13d75ec951519a1a21938c533767b175db0c8e Mon Sep 17 00:00:00 2001 From: Richard Franklin Date: Sat, 24 Aug 2024 17:08:25 -0700 Subject: [PATCH 20/21] Deprecating dict returns for on_item_batch --- docs/guides.rst | 4 ++-- graphbook/steps/base.py | 36 +++++++++++++++++++++++++----------- graphbook/web.py | 8 ++++---- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/docs/guides.rst b/docs/guides.rst index 70ec2a7..51a26bf 100644 --- a/docs/guides.rst +++ b/docs/guides.rst @@ -503,8 +503,8 @@ Then, create a new BatchStep class that uses the RMBG-1.4 model to remove the ba masks = [] masks.append({"value": path, "type": "image"}) note["masks"] = masks - - return {"removed_bg": removed_bg} + + return removed_bg This node will generate masks of the foreground using the RMBG-1.4 model and output the resulting mask as images by saving them to disk. See that there is one notable difference in RemoveBackground compared to PokemonClassifier. diff --git a/graphbook/steps/base.py b/graphbook/steps/base.py index 256085e..e027555 100644 --- a/graphbook/steps/base.py +++ b/graphbook/steps/base.py @@ -3,8 +3,11 @@ from graphbook.dataloading import Dataloader from ..utils import transform_function_string from graphbook import Note +import warnings +warnings.simplefilter("default", DeprecationWarning) + StepOutput = Dict[str, List[Note]] """A dict mapping of output slot to Note list. Every Step outputs a StepOutput.""" @@ -338,7 +341,7 @@ def dump_fn(data: any, output_dir: str, uid: int): output_dir (str): The output directory uid (int): A unique identifier for the data """ - raise NotImplementedError("dump_fn must be implemented for BatchStep") + raise NotImplementedError("dump_fn must be implemented for BatchStep when dumping outputs") def handle_batch(self, batch: StepData): items, notes, completed = batch @@ -349,14 +352,23 @@ def handle_batch(self, batch: StepData): ] data_dump = self.on_item_batch(tensors, items, notes) if data_dump is not None: - for k, v in data_dump.items(): - if len(notes) != len(v): - self.logger.log( - f"Unexpected number of notes ({len(notes)}) does not match returned outputs ({len(v)}). Will not write outputs!" - ) - else: - for note, out in zip(notes, v): - self.dump_data(note, out) + if isinstance(data_dump, dict): + # Dict returns are deprecated + warnings.warn( + "dict returns for on_item_batch are deprecated and will be removed in a future version. Please return a list of your parameters to provide to dump_fn instead.", + DeprecationWarning, + ) + for k, v in data_dump.items(): + if len(notes) != len(v): + self.logger.log( + f"Unexpected number of notes ({len(notes)}) does not match returned outputs ({len(v)}). Will not write outputs!" + ) + else: + for note, out in zip(notes, v): + self.dump_data(note, out) + else: + for note, out in zip(notes, data_dump): + self.dump_data(note, out) for note in completed: self.dumped_item_holders.set_completed(note) @@ -374,17 +386,19 @@ def handle_completed_notes(self): output[output_key].append(note) return output - def on_item_batch(self, tensors, items, notes): + def on_item_batch(self, tensors, items, notes) -> List[any] | None: """ Called when B items are loaded into PyTorch tensors and are ready to be processed where B is *batch_size*. This is meant to be overriden by subclasses. - Args: tensors (List[torch.Tensor]): The list of loaded tensors of length B items (List[any]): The list of anys of length B associated with tensors. This list has the same order as tensors does along the batch dimension notes (List[Note]): The list of Notes of length B associated with tensors. This list has the same order as tensor does along the batch dimension + + Returns: + List[any] | None: The output data to be dumped as a list of parameters to be passed to dump_fn. If None is returned, nothing will be dumped. """ pass diff --git a/graphbook/web.py b/graphbook/web.py index 972f65b..1826a59 100644 --- a/graphbook/web.py +++ b/graphbook/web.py @@ -245,7 +245,7 @@ async def get_resource_docstring(request: web.Request): @routes.get("/fs") @routes.get(r"/fs/{path:.+}") - def get(request: web.Request): + async def get(request: web.Request): path = request.match_info.get("path", "") fullpath = osp.join(abs_root_path, path) assert fullpath.startswith( @@ -352,7 +352,7 @@ async def put(request: web.Request): return web.json_response({}, status=201) @routes.delete("/fs/{path:.+}") - def delete(request): + async def delete(request): path = request.match_info.get("path") fullpath = osp.join(root_path, path) assert fullpath.startswith( @@ -378,9 +378,9 @@ def delete(request): ) async def _async_start(self): - runner = web.AppRunner(self.app) + runner = web.AppRunner(self.app, shutdown_timeout=0.5) await runner.setup() - site = web.TCPSite(runner, self.host, self.port, shutdown_timeout=0.5) + site = web.TCPSite(runner, self.host, self.port) loop = asyncio.get_running_loop() await site.start() await loop.run_in_executor(None, self.view_manager.start) From 9f71c0261df51b7a2bf001b3f897f29a8df8b303 Mon Sep 17 00:00:00 2001 From: Richard Franklin Date: Sat, 24 Aug 2024 17:32:43 -0700 Subject: [PATCH 21/21] Incrementing version number --- pyproject.toml | 2 +- web/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2c4cec8..aecea33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "graphbook" -version = "0.4.3" +version = "0.5.0" authors = ["Richard Franklin "] description = "An extensible ML workflow framework built for data scientists and ML engineers." keywords = ["ml", "workflow", "framework", "pytorch", "data science", "machine learning", "ai"] diff --git a/web/package.json b/web/package.json index b677fe4..f7f6124 100644 --- a/web/package.json +++ b/web/package.json @@ -1,7 +1,7 @@ { "name": "graphbook-web", "private": true, - "version": "0.4.3", + "version": "0.5.0", "type": "module", "scripts": { "dev": "vite",