🐣 Birth of Graphbook

.github/workflows/static.yml

@@ -0,0 +1,46 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy Graphbook Docs
+
+on:
+  push:
+    branches: ["main"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Build Graphbook Docs
+        run: |
+          pip install -r docs/requirements.txt
+          make -C docs html
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload entire repository
+          path: 'docs/_build/html'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -0,0 +1,39 @@
+# Python
+.venv
+__pycache__
+
+# Docs
+docs/_build
+
+# Config, Inputs, Outputs
+*.json
+!package*.json
+*.jsonl
+*.csv
+/workflow
+/media
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Graphbook AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,15 @@
+# Builds graphbook by installing dependencies and building the web app
+# Usage: make [web|docs]
+
+# Note: Python dependencies are installed in the requirements.txt and are left to the user to install
+
+.PHONY: all web docs
+
+all: web docs
+
+web:
+	cd web; npm install
+	cd web; npm run build
+
+docs:
+	$(MAKE) -C docs html

README.md

@@ -1,4 +1,29 @@
-# graphbook
-The interactive and extensible editor for ML workflows
+# Graphbook
+Graphbook is an extensible and interactive ML workflow editor that allows you to build monitorable data processing pipelines powered with ML. It can be used to prepare training data for custom ML models, experiment with custom trained or off-the-shelf models, and to build ML-based ETL applications. Custom nodes can be built in Python, and Graphbook will behave like a framework and call lifecycle methods on those nodes.
 
-Graphbook is coming soon. Keep a watch on the repo for updates.
+## Current Features
+- Graph-based visual editor to experiment and create complex ML workflows
+- Internal Python code editor
+- Executable graph, subgraphs, and individual nodes
+- Only re-executes parts of the workflow that changes between executions
+- Node logging and output views for monitoring in the UI
+- Custom buildable nodes with Python
+- Automatic batching for Pytorch tensors
+- Multiprocessing I/O to and from disk and network
+- Customizeable multiprocessing functions
+- Basic nodes for filtering, loading, and saving outputs
+
+## Getting Started
+View the docs to learn how to install and get started with Graphbook.
+
+## Collaboration Guide
+This is a guide on how to get started developing Graphbook. If you are simply using Graphbook, view the [Getting Started](#getting-started) section.
+1. Install python>=3.11.2. There is a known bug in python 3.10.
+1. Clone and `cd graphbook/`
+1. (Optional) Create your venv
+1. `pip install -r requirements.txt`
+1. `pip install -r docs/requirements.txt`
+1. `python src/server.py`
+1. `cd web`
+1. `npm install`
+1. `npm run dev`

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,37 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'Graphbook'
+copyright = '2024, Graphbook AI'
+author = 'Graphbook AI'
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../src'))
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = ['sphinx.ext.autodoc',
+              'sphinx.ext.coverage',
+              'sphinx.ext.napoleon',
+              'sphinx_rtd_theme']
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+html_theme_options = {
+  "navigation_depth": -1
+}
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'sphinx_rtd_theme'
+html_static_path = ['_static']

docs/contributing.rst

@@ -0,0 +1,3 @@
+Contributing
+*****************
+If you would like to contribute to the development of Graphbook, please fork the repo and submit your PR.

docs/examples.rst

@@ -0,0 +1,3 @@
+Examples
+*****************
+Coming soon.

docs/getting_started.rst

@@ -0,0 +1,18 @@
+Getting Started
+*****************
+
+1. Create a directory
+    This directory will contain your workflows. Create it and cd into it
+
+2. Run the server in that directory
+    Move to the directory that you want to create your workflows and custom nodes in and run the following script from the graphbook repository:
+
+.. code-block:: bash
+
+    <path_to_graphbook_repo>/scripts/graphbook
+
+3. Visit the site
+    Open your browser and visit http://localhost:8007
+
+4. Start your workflows!
+    We are currently working on examples and tutorials to help you get started with Graphbook. Stay tuned!

docs/index.rst

@@ -0,0 +1,16 @@
+Graphbook
+=====================================
+Graphbook is the open-source, interactive, and extensible editor for ML workflows. It is designed to help you build, visualize, and monitor your ML workflows with ease.
+
+.. _Graphbook: https://www.graphbook.ai
+
+.. toctree::
+   :caption: Graphbook
+   :maxdepth: 2
+   :hidden:
+
+   installing
+   getting_started
+   reference
+   examples
+   contributing

docs/installing.rst

@@ -0,0 +1,19 @@
+Installing
+*****************
+We are currently working on docker containers and binaries, so that users can quickly get started. But for now, to install Graphbook from source:
+
+#. Clone the repository and cd into the root of the repo
+
+#. Install the web frontend
+
+   .. code-block:: bash
+        
+        make web
+
+#. Optionally, create and activate your venv
+
+#. Install the requirements
+
+   .. code-block:: bash
+
+        pip install -r requirements.txt

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/reference.rst

@@ -0,0 +1,13 @@
+Reference
+*****************
+
+API documentation
+
+.. automodule:: graphbook.steps.base
+   :members:
+
+.. automodule:: graphbook.steps.io
+   :members:
+
+.. automodule:: graphbook.steps.hf_transformers
+   :members:

docs/requirements.txt

@@ -0,0 +1,8 @@
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.5
+sphinxcontrib-jquery==4.1
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.7
+sphinxcontrib-serializinghtml==1.1.10
+sphinx-rtd-theme==2.0.0

pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "graphbook"
+version = "0.0.0"

requirements.txt

@@ -0,0 +1,35 @@
+aiohttp==3.9.3
+aiosignal==1.3.1
+async-timeout==4.0.3
+attrs==23.2.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+dill==0.3.8
+filelock==3.13.1
+frozenlist==1.4.1
+fsspec==2024.3.1
+huggingface-hub==0.22.2
+idna==3.6
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+mpmath==1.3.0
+multidict==6.0.5
+networkx==3.2.1
+numpy==1.26.4
+packaging==24.0
+pillow==10.3.0
+python-magic==0.4.27
+PyYAML==6.0.1
+regex==2024.4.16
+requests==2.31.0
+safetensors==0.4.3
+sympy==1.12
+tokenizers==0.19.1
+torch==2.2.2
+torchvision==0.17.2
+tqdm==4.66.2
+transformers==4.40.1
+typing_extensions==4.10.0
+urllib3==2.2.1
+watchdog==4.0.0
+yarl==1.9.4

scripts/graphbook

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+REPO_PATH=$(dirname $(realpath $0))/..
+SCRIPT_DIR=$0
+SERVER_PATH=$REPO_PATH/src/server.py
+WEB_PATH=$REPO_PATH/web
+WEB_BUILD_PATH=$WEB_PATH/dist
+
+python $SERVER_PATH --web --web_dir $WEB_BUILD_PATH --workflow_dir . --nodes_dir ./nodes $@