Add "get column usage" section to docs

vega · Nov 6, 2024 · 8a3d550 · 8a3d550
1 parent 1b15042
commit 8a3d550
Show file tree

Hide file tree

Showing 15 changed files with 542 additions and 3,524 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [workspace]
 resolver = "2"
-members = [ "vegafusion-common", "vegafusion-core", "vegafusion-runtime", "vegafusion-python", "vegafusion-wasm", "vegafusion-server",]
+members = [ "vegafusion-common", "vegafusion-core", "vegafusion-runtime", "vegafusion-python", "vegafusion-wasm", "vegafusion-server", "examples/rust-examples",]
 
 [workspace.dependencies]
 deterministic-hash = "1.0.1"

diff --git a/docs/source/column_usage.md b/docs/source/column_usage.md
@@ -0,0 +1,17 @@
+# Get Column Usage
+VegaFusion provides a function for introspecting a Vega specification and determining which columns are referenced from each root dataset. A root dataset is one defined at the top-level of the spec that includes a `url` or `values` properties. This is useful in contexts where it's more efficient to minimize the number of columns provided to the Vega specification. For example, the Python library uses this function to determine how to downsample the input DataFrame columns prior to converting to Arrow.
+
+When VegaFusion cannot precisely determine which columns are referenced from each root dataset, this function returns `None` or `null` for the corresponding dataset.
+
+## Python
+```{eval-rst}
+.. autofunction:: vegafusion.get_column_usage
+```
+
+See [column_usage.py](https://github.com/vega/vegafusion/tree/v2/examples/python-examples/column_usage.py) for a complete example.
+
+## Rust
+See [column_usage.rs](https://github.com/vega/vegafusion/tree/v2/examples/rust-examples/examples/column_usage.rs) for a complete example.
+
+## JavaScript
+See the [Editor Demo](https://github.com/vega/vegafusion/tree/v2/examples/editor-demo/src/index.js) for example usage of the `getColumnUsage` function in the `vegafusion-wasm` package.
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -5,6 +5,8 @@
 extensions = [
     'myst_parser',
     'sphinx_copybutton',
+    'sphinx.ext.autodoc',
+    'sphinx.ext.napoleon',
 ]
 
 # Theme settings

diff --git a/docs/source/index.md b/docs/source/index.md
@@ -12,3 +12,10 @@ The VegaFusion project provides Rust, Python, and JavaScript libraries for analy
 :::{note}
 If you've arrived here looking for information on how to scale Vega-Altair visualizations to support larger datasets, see the Vega-Altair documentation on the [`"vegafusion"` data transformer](https://altair-viz.github.io/user_guide/large_datasets.html#vegafusion-data-transformer).
 :::
+
+```{toctree}
+:maxdepth: 2
+:caption: Contents
+
+column_usage
+```
diff --git a/examples/editor-demo/src/index.html b/examples/editor-demo/src/index.html
@@ -24,6 +24,9 @@ <h1>VegaFusion</h1>
           <li class="nav-item">
             <a href="#comm-plan" class="nav-link" data-bs-toggle="tab">Comm Plan</a>
           </li>
+          <li class="nav-item">
+            <a href="#column-usage" class="nav-link" data-bs-toggle="tab">Column Usage</a>
+          </li>
         </ul>
         <div class="tab-content">
           <div class="tab-pane show active" id="editor">
@@ -38,6 +41,9 @@ <h1>VegaFusion</h1>
           <div class="tab-pane" id="comm-plan">
             <div id="comm-plan-monaco" class="monaco-editor"></div>
           </div>
+          <div class="tab-pane" id="column-usage">
+            <div id="column-usage-monaco" class="monaco-editor"></div>
+          </div>
         </div>
       </div>
     </div>

diff --git a/examples/editor-demo/src/index.js b/examples/editor-demo/src/index.js
@@ -1,4 +1,4 @@
-const { vegaFusionEmbed, makeGrpcSendMessageFn } = await import("vegafusion-wasm");
+const { vegaFusionEmbed, getColumnUsage, makeGrpcSendMessageFn } = await import("vegafusion-wasm");
 
 import * as Monaco from 'monaco-editor/esm/vs/editor/editor.api';
 import _ from "lodash"
@@ -45,6 +45,14 @@ async function init() {
         readOnly: true,
     });
 
+    let column_usage_monaco = Monaco.editor.create(document.getElementById('column-usage-monaco'), {
+        value: "",
+        language: 'json',
+        theme: 'vs-dark',
+        automaticLayout: true,
+        readOnly: true,
+    });
+
     // Add checkbox for toggling VegaFusion server
     const container = document.createElement('div');
     container.className = 'pb-3';  // Add padding-bottom
@@ -79,6 +87,8 @@ async function init() {
                 },
             };
 
+            let spec = editor.getValue();
+
             let chart_handle;
             if (serverCheckbox.checked) {
                 const hostname = 'http://127.0.0.1:50051';
@@ -88,7 +98,7 @@ async function init() {
 
                     chart_handle = await vegaFusionEmbed(
                         element,
-                        editor.getValue(),
+                        spec,
                         config,
                         send_message_grpc,
                     );
@@ -105,19 +115,24 @@ async function init() {
             } else {
                 chart_handle = await vegaFusionEmbed(
                     element,
-                    editor.getValue(),
+                    spec,
                     config,
                 );
             }
 
+            // Get column usage
+            const usage = getColumnUsage(spec);
+
             server_spec_monaco.setValue(JSON.stringify(chart_handle.serverSpec(), null, 2));
             client_spec_monaco.setValue(JSON.stringify(chart_handle.clientSpec(), null, 2));
             comm_plan_monaco.setValue(JSON.stringify(chart_handle.commPlan(), null, 2));
+            column_usage_monaco.setValue(JSON.stringify(usage, null, 2));
         } catch (e) {
             console.error("Failed to render spec");
             server_spec_monaco.setValue("");
             client_spec_monaco.setValue("");
             comm_plan_monaco.setValue("");
+            column_usage_monaco.setValue("");
             console.log(e);
             return;
         }

diff --git a/examples/python-examples/column_usage.py b/examples/python-examples/column_usage.py
@@ -0,0 +1,125 @@
+import json
+from typing import Any
+
+from vegafusion import get_column_usage
+
+
+def main():
+    spec = get_spec()
+    column_usage = get_column_usage(spec)
+    print(json.dumps(column_usage, indent=2))
+
+    assert column_usage == {
+        "source": ["Acceleration", "Horsepower", "Miles_per_Gallon"]
+    }
+
+
+def get_spec() -> dict[str, Any]:
+    spec_str = """
+    {
+        "$schema": "https://vega.github.io/schema/vega/v5.json",
+        "description": "A basic scatter plot example depicting automobile statistics.",
+        "width": 200,
+        "height": 200,
+        "padding": 5,
+
+        "data": [
+            {
+            "name": "source",
+            "url": "data/cars.json",
+            "transform": [
+                {
+                "type": "filter",
+                "expr": "datum['Horsepower'] != null && datum['Miles_per_Gallon'] != null && datum['Acceleration'] != null"
+                }
+            ]
+            }
+        ],
+
+        "scales": [
+            {
+            "name": "x",
+            "type": "linear",
+            "round": true,
+            "nice": true,
+            "zero": true,
+            "domain": {"data": "source", "field": "Horsepower"},
+            "range": "width"
+            },
+            {
+            "name": "y",
+            "type": "linear",
+            "round": true,
+            "nice": true,
+            "zero": true,
+            "domain": {"data": "source", "field": "Miles_per_Gallon"},
+            "range": "height"
+            },
+            {
+            "name": "size",
+            "type": "linear",
+            "round": true,
+            "nice": false,
+            "zero": true,
+            "domain": {"data": "source", "field": "Acceleration"},
+            "range": [4,361]
+            }
+        ],
+
+        "axes": [
+            {
+            "scale": "x",
+            "grid": true,
+            "domain": false,
+            "orient": "bottom",
+            "tickCount": 5,
+            "title": "Horsepower"
+            },
+            {
+            "scale": "y",
+            "grid": true,
+            "domain": false,
+            "orient": "left",
+            "titlePadding": 5,
+            "title": "Miles_per_Gallon"
+            }
+        ],
+
+        "legends": [
+            {
+            "size": "size",
+            "title": "Acceleration",
+            "format": "s",
+            "symbolStrokeColor": "#4682b4",
+            "symbolStrokeWidth": 2,
+            "symbolOpacity": 0.5,
+            "symbolType": "circle"
+            }
+        ],
+
+        "marks": [
+            {
+            "name": "marks",
+            "type": "symbol",
+            "from": {"data": "source"},
+            "encode": {
+                "update": {
+                "x": {"scale": "x", "field": "Horsepower"},
+                "y": {"scale": "y", "field": "Miles_per_Gallon"},
+                "size": {"scale": "size", "field": "Acceleration"},
+                "shape": {"value": "circle"},
+                "strokeWidth": {"value": 2},
+                "opacity": {"value": 0.5},
+                "stroke": {"value": "#4682b4"},
+                "fill": {"value": "transparent"}
+                }
+            }
+            }
+        ]
+    }
+    """
+    return json.loads(spec_str)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/rust-examples/Cargo.toml b/examples/rust-examples/Cargo.toml
@@ -0,0 +1,8 @@
+[package]
+name = "rust-examples"
+version = "0.1.0"
+edition = "2021"
+
+[dev-dependencies]
+serde_json = { workspace = true }
+vegafusion-core = { path = "../../vegafusion-core" }