From 7450fd0c3e967c7130ec52c430899af0141d3114 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 12 Nov 2024 10:16:21 -0500 Subject: [PATCH] v2-docs: Add embed section, general review and cleanup (#538) * Add JavaScript embed docs * doc review * fmt --- docs/source/_static/custom-icon.js | 18 ++++++++++++++++ docs/source/about/related_projects.md | 4 ++++ docs/source/about/technology.md | 2 +- docs/source/conf.py | 21 ++++++++++++++++++- docs/source/features/embed.md | 8 +++++++ docs/source/features/features.md | 1 + docs/source/features/grpc.md | 4 ++-- docs/source/features/transform_spec.md | 2 +- .../vega_coverage/supported_transforms.md | 2 +- vegafusion-python/vegafusion/runtime.py | 8 +++---- vegafusion-wasm/README.md | 8 +++++-- 11 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 docs/source/_static/custom-icon.js create mode 100644 docs/source/features/embed.md diff --git a/docs/source/_static/custom-icon.js b/docs/source/_static/custom-icon.js new file mode 100644 index 00000000..1106011a --- /dev/null +++ b/docs/source/_static/custom-icon.js @@ -0,0 +1,18 @@ +/******************************************************************************* + * Set a custom icon for pypi as it's not available in the fa built-in brands + * + * Taken from PyData Sphinx Theme's own documentation + */ +FontAwesome.library.add( + (faListOldStyle = { + prefix: "fa-custom", + iconName: "pypi", + icon: [ + 17.313, // viewBox width + 19.807, // viewBox height + [], // ligature + "e001", // unicode codepoint - private use area + "m10.383 0.2-3.239 1.1769 3.1883 1.1614 3.239-1.1798zm-3.4152 1.2411-3.2362 1.1769 3.1855 1.1614 3.2369-1.1769zm6.7177 0.00281-3.2947 1.2009v3.8254l3.2947-1.1988zm-3.4145 1.2439-3.2926 1.1981v3.8254l0.17548-0.064132 3.1171-1.1347zm-6.6564 0.018325v3.8247l3.244 1.1805v-3.8254zm10.191 0.20931v2.3137l3.1777-1.1558zm3.2947 1.2425-3.2947 1.1988v3.8254l3.2947-1.1988zm-8.7058 0.45739c0.00929-1.931e-4 0.018327-2.977e-4 0.027485 0 0.25633 0.00851 0.4263 0.20713 0.42638 0.49826 1.953e-4 0.38532-0.29327 0.80469-0.65542 0.93662-0.36226 0.13215-0.65608-0.073306-0.65613-0.4588-6.28e-5 -0.38556 0.2938-0.80504 0.65613-0.93662 0.068422-0.024919 0.13655-0.038114 0.20156-0.039466zm5.2913 0.78369-3.2947 1.1988v3.8247l3.2947-1.1981zm-10.132 1.239-3.2362 1.1769 3.1883 1.1614 3.2362-1.1769zm6.7177 0.00213-3.2926 1.2016v3.8247l3.2926-1.2009zm-3.4124 1.2439-3.2947 1.1988v3.8254l3.2947-1.1988zm-6.6585 0.016195v3.8275l3.244 1.1805v-3.8254zm16.9 0.21143-3.2947 1.1988v3.8247l3.2947-1.1981zm-3.4145 1.2411-3.2926 1.2016v3.8247l3.2926-1.2009zm-3.4145 1.2411-3.2926 1.2016v3.8247l3.2926-1.2009zm-3.4124 1.2432-3.2947 1.1988v3.8254l3.2947-1.1988zm-6.6585 0.019027v3.8247l3.244 1.1805v-3.8254zm13.485 1.4497-3.2947 1.1988v3.8247l3.2947-1.1981zm-3.4145 1.2411-3.2926 1.2016v3.8247l3.2926-1.2009zm2.4018 0.38127c0.0093-1.83e-4 0.01833-3.16e-4 0.02749 0 0.25633 0.0085 0.4263 0.20713 0.42638 0.49826 1.97e-4 0.38532-0.29327 0.80469-0.65542 0.93662-0.36188 0.1316-0.65525-0.07375-0.65542-0.4588-1.95e-4 -0.38532 0.29328-0.80469 0.65542-0.93662 0.06842-0.02494 0.13655-0.03819 0.20156-0.03947zm-5.8142 0.86403-3.244 1.1805v1.4201l3.244 1.1805z", // svg path (https://simpleicons.org/icons/pypi.svg) + ], + }) + ); \ No newline at end of file diff --git a/docs/source/about/related_projects.md b/docs/source/about/related_projects.md index 39ce1c82..01923d15 100644 --- a/docs/source/about/related_projects.md +++ b/docs/source/about/related_projects.md @@ -11,6 +11,8 @@ These workflows were not supported by the initial version of VegaFusion, but sup `altair-transform` does not support evaluating transforms on the server in interactive workflows like linked histogram brushing, which was the initial focus of VegaFusion. +`altair-transform` is not under active development. + ## [`ibis-vega-transform`](https://github.com/Quansight/ibis-vega-transform) `ibis-vega-transform` is a Python library and JupyterLab extension developed by [Quansight](https://www.quansight.com/). It translates pipelines of Vega transforms into [Ibis](https://ibis-project.org/) query expressions, which can then be evaluated with a variety of Ibis database backends (in particular, OmniSci). @@ -21,3 +23,5 @@ In contrast to the Planner approach used by VegaFusion, `ibis-vega-transform` re An advantage of this approach is that the Vega JavaScript library remains in control of the entire specification so the external `ibis-vega-transform` library does not need to maintain an independent task graph in order to support interactivity. A downside of this approach is that the result of every transform pipeline must be sent back to the client and be stored in the Vega dataflow graph. Often times this is not a problem, because the transform pipeline includes an aggregation stage that significantly reduces the dataset size. However, sometimes the result of a transform pipeline is quite large, but it is only used as input to other transform pipelines. In this case, it is advantageous to keep the large intermediary result cached on the server and to not send it to the client at all. This use case is one of the reasons that VegaFusion uses the Planner+Runtime architecture described previously. Currently, VegaFusion implements all of its transform logic in the Python process (with efficient multi-threading) and has no capability to connect to external data providers like databases. This is certainly a desirable capability, and may be enabled in VegaFusion by the [datafusion-federation](https://github.com/datafusion-contrib/datafusion-federation) project. + +`ibis-vega-transform` is not under active development. diff --git a/docs/source/about/technology.md b/docs/source/about/technology.md index 3cc5cda9..7073c35b 100644 --- a/docs/source/about/technology.md +++ b/docs/source/about/technology.md @@ -5,4 +5,4 @@ VegaFusion uses a fairly diverse technology stack. The planner and runtime are b The Task Graph specifications are defined as protocol buffer messages. The [prost](https://github.com/tokio-rs/prost) library is used to generate Rust data structures from these protocol buffer messages. When Arrow tables appear as task graph root values, they are serialized inside the protocol buffer specification using the [Apache Arrow IPC format](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc). The binary representation of the task graph protocol buffer message is what is transferred across the Jupyter Comms protocol. ## DataFusion integration -[Apache Arrow DataFusion](https://github.com/apache/arrow-datafusion) is an SQL compatible query engine that integrates with the Rust implementation of Apache Arrow. VegaFusion uses DataFusion to implement many of the Vega transforms, and it compiles the Vega expression language directly into the DataFusion expression language. In addition to being very fast, a particularly powerful characteristic of DataFusion is that it provides many interfaces that can be extended with custom Rust logic. For example, VegaFusion defines a few custom UDFs that are designed to implement the precise semantics of the Vega transforms and the Vega expression language. +[Apache DataFusion](https://github.com/apache/datafusion) is an SQL compatible query engine that integrates with the Rust implementation of Apache Arrow. VegaFusion uses DataFusion to implement many of the Vega transforms, and it compiles the Vega expression language directly into the DataFusion expression language. In addition to being very fast, a particularly powerful characteristic of DataFusion is that it provides many interfaces that can be extended with custom Rust logic. For example, VegaFusion defines a few custom UDFs that are designed to implement the precise semantics of the Vega transforms and the Vega expression language. diff --git a/docs/source/conf.py b/docs/source/conf.py index 6d46937c..bf116aab 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,11 +17,30 @@ html_static_path = ['_static'] html_logo = "_static/VegaFusionLogo-Color.svg" html_favicon = "_static/favicon.ico" - +html_theme_options = { + "icon_links": [ + { + "name": "Twitter", + "url": "https://twitter.com/vegafusion_io", + "icon": "fa-brands fa-twitter", + }, + { + "name": "GitHub", + "url": "https://github.com/vega/vegafusion", + "icon": "fa-brands fa-github", + }, + { + "name": "PyPI", + "url": "https://pypi.org/project/vegafusion/", + "icon": "fa-custom fa-pypi", + } + ], +} # Add custom CSS html_css_files = [ 'custom.css', ] +html_js_files = ["custom-icon.js"] _social_img = "https://vegafusion.io/_static/vegafusion_social.png" _description = "VegaFusion provides serverside scaling for Vega visualizations" diff --git a/docs/source/features/embed.md b/docs/source/features/embed.md new file mode 100644 index 00000000..4c12645f --- /dev/null +++ b/docs/source/features/embed.md @@ -0,0 +1,8 @@ +# JavaScript Embed +The [`vegafusion-wasm`](https://www.npmjs.com/package/vegafusion-wasm) NPM package provides a JavaScript API for embedding Vega charts backed by VegaFusion in web applications. These charts can be connected to a local VegaFusion runtime compiled to WebAssembly, or a remote VegaFusion server over gRPC-Web. + +## Usage +See the [vegafusion-wasm README](https://github.com/vega/vegafusion/blob/v2/vegafusion-wasm/README.md) for usage instructions. + +## Demo +See [editor demo](https://github.com/vega/vegafusion/tree/v2/examples/editor-demo) for a complete example. diff --git a/docs/source/features/features.md b/docs/source/features/features.md index df3365f8..bdc789d4 100644 --- a/docs/source/features/features.md +++ b/docs/source/features/features.md @@ -14,5 +14,6 @@ transform_extract chart_state inline_datasets grpc +embed vega-lite ``` diff --git a/docs/source/features/grpc.md b/docs/source/features/grpc.md index 551071f3..9805a64c 100644 --- a/docs/source/features/grpc.md +++ b/docs/source/features/grpc.md @@ -1,5 +1,5 @@ # gRPC -The VegaFusion Runtime can run as a [gRPC](https://grpc.io/) service, which makes it possible for multiple clients to connect to the same runtime, and share a cache. This also makes it possible for the Runtime to reside on a different host than the client. +The VegaFusion Runtime can run as a [gRPC](https://grpc.io/) service, which makes it possible for multiple clients to connect to the same runtime, and share a cache (See [How it Works](../about/how_it_works) for more details). This also makes it possible for the Runtime to reside on a different host than the client. :::{warning} VegaFusion's gRPC server does not currently support authentication, and chart specifications may reference the local file system of the machine running the server. It is not currently recommended to use VegaFusion server with untrusted Vega specifications unless other measures are taken to isolate the service. @@ -35,7 +35,7 @@ See [grpc.py](https://github.com/vega/vegafusion/tree/v2/examples/python-example ## Rust The `GrpcVegaFusionRuntime` struct is an alternative to the `VegaFusionRuntime` struct that provides the same interface, but connects to a VegaFusion Server. -See [grpc.rs](https://github.com/vega/vegafusion/tree/v2/examples/rust-examples/grpc.rs) for a complete example. +See [grpc.rs](https://github.com/vega/vegafusion/tree/v2/examples/rust-examples/examples/grpc.rs) for a complete example. ## JavaScript The `vegafusion-wasm` package can connect to an instance of VegaFusion Server over [gRPC-Web](https://github.com/grpc/grpc-web). diff --git a/docs/source/features/transform_spec.md b/docs/source/features/transform_spec.md index 4800f73d..7055528f 100644 --- a/docs/source/features/transform_spec.md +++ b/docs/source/features/transform_spec.md @@ -5,7 +5,7 @@ VegaFusion can be used to evaluate datasets in a Vega spec, remove unused column This is the foundation of Vega-Altair's [``"vegafusion"`` data transformer](https://altair-viz.github.io/user_guide/large_datasets.html#vegafusion-data-transformer) when used with the default HTML or static image renderers. :::{warning} -The pre-transform process will, by default, preserve the interactive behavior of the input Vega specification. For interactive charts that perform filtering, this may result in the generation of a spec containing the full input dataset. If interactivity does not need to be preserved (e.g. if the resulting chart is used in a static context) then the ``preserve_interactivity`` option should be set to False. If interactivity is needed, then the Chart State workflow may be more appropriate. +The pre-transform process will, by default, preserve the interactive behavior of the input Vega specification. For interactive charts that perform filtering, this may result in the generation of a spec containing the full input dataset. If interactivity does not need to be preserved (e.g. if the resulting chart is used in a static context) then the ``preserve_interactivity`` option should be set to False. If interactivity is needed, then the [Chart State](./chart_state) workflow may be more appropriate. ::: ## Python diff --git a/docs/source/vega_coverage/supported_transforms.md b/docs/source/vega_coverage/supported_transforms.md index e74e8957..f7d53790 100644 --- a/docs/source/vega_coverage/supported_transforms.md +++ b/docs/source/vega_coverage/supported_transforms.md @@ -1,6 +1,6 @@ # Supported Transforms -VegaFusion implements a subset of [Vega's transforms](https://vega.github.io/vega/). Below is a detailed breakdown of transform support status. +VegaFusion implements a subset of [Vega's transforms](https://vega.github.io/vega/docs/transforms/). Below is a detailed breakdown of transform support status. :::{note} diff --git a/vegafusion-python/vegafusion/runtime.py b/vegafusion-python/vegafusion/runtime.py index 3b58abd1..a010537e 100644 --- a/vegafusion-python/vegafusion/runtime.py +++ b/vegafusion-python/vegafusion/runtime.py @@ -452,8 +452,6 @@ def pre_transform_datasets( Extract the fully evaluated form of the requested datasets from a Vega specification. - Extracts datasets as pandas DataFrames. - Args: spec: A Vega specification dict or JSON string. datasets: A list with elements that are either: @@ -606,9 +604,9 @@ def pre_transform_extract( """ Evaluate supported transforms in an input Vega specification. - Produces a new specification with small pre-transformed datasets (under 100 - rows) included inline and larger inline datasets (20 rows or more) extracted - into pyarrow tables. + Produces a new specification with small pre-transformed datasets + (under ``extract_threshold`` rows) included inline and larger inline + datasets (``extract_threshold`` rows or more) extracted into arrow tables. Args: spec: A Vega specification dict or JSON string. diff --git a/vegafusion-wasm/README.md b/vegafusion-wasm/README.md index f2fdc6e7..b3b38fb5 100644 --- a/vegafusion-wasm/README.md +++ b/vegafusion-wasm/README.md @@ -3,7 +3,7 @@ Wasm library for interfacing with VegaFusion. For more information, see the VegaFusion repository at https://github.com/hex-inc/vegafusion -## Usage +## gRPC-WebUsage Example usage with the VegaFusion server running locally on port 50051, with grpc-enabled: ``` @@ -31,11 +31,15 @@ let config = { let chart_handle = await vegaFusionEmbed( element, spec, // Replace with your Vega spec - send_message_grpc, config, + send_message_grpc, ); ``` +## Embedded Runtime Usage + +To use an embedded VegaFusion Runtime compiled with Web Assembly, simply omit the `send_message_grpc` argument from the `vegaFusionEmbed` call. In this configuration, the chart is fully self-contained and does not require a VegaFusion server. + ## webpack config To use this library in a webpack project, you need to add the following [experimental options](https://webpack.js.org/configuration/experiments/) to your webpack config: