From bdc721d73d00e4d3b5c41a3ba6a4aa9038eb09b5 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 16 Dec 2023 09:50:08 -0500 Subject: [PATCH 1/2] Support threshold aggregation example --- vegafusion-core/src/planning/extract.rs | 3 +- vegafusion-core/src/planning/stitch.rs | 1 + vegafusion-core/src/spec/signal.rs | 3 + vegafusion-core/src/spec/visitors.rs | 4 +- .../aggregate_with_threshold.comm_plan.json | 36 +++ .../aggregate_with_threshold.updates.json | 18 ++ .../custom/aggregate_with_threshold.vg.json | 267 ++++++++++++++++++ .../tests/test_image_comparison.rs | 3 +- 8 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.comm_plan.json create mode 100644 vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.updates.json create mode 100644 vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.vg.json diff --git a/vegafusion-core/src/planning/extract.rs b/vegafusion-core/src/planning/extract.rs index 6419ad9e3..df95b27bb 100644 --- a/vegafusion-core/src/planning/extract.rs +++ b/vegafusion-core/src/planning/extract.rs @@ -176,7 +176,8 @@ impl<'a> MutChartVisitor for ExtractServerDependenciesVisitor<'a> { let scoped_signal_var = (Variable::new_signal(&signal.name), Vec::from(scope)); if self.supported_vars.contains_key(&scoped_signal_var) { // Move signal to server - let server_signal = signal.clone(); + let mut server_signal = signal.clone(); + server_signal.bind = None; if scope.is_empty() { self.server_spec.signals.push(server_signal) } else { diff --git a/vegafusion-core/src/planning/stitch.rs b/vegafusion-core/src/planning/stitch.rs index bf4a18b10..ded556c6a 100644 --- a/vegafusion-core/src/planning/stitch.rs +++ b/vegafusion-core/src/planning/stitch.rs @@ -102,6 +102,7 @@ fn make_stub( update: None, value: MissingNullOrValue::from(stub_value), on: vec![], + bind: None, extra: Default::default(), }; diff --git a/vegafusion-core/src/spec/signal.rs b/vegafusion-core/src/spec/signal.rs index 688be16e2..ea5bc4efc 100644 --- a/vegafusion-core/src/spec/signal.rs +++ b/vegafusion-core/src/spec/signal.rs @@ -23,6 +23,9 @@ pub struct SignalSpec { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub on: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub bind: Option, + #[serde(flatten)] pub extra: HashMap, } diff --git a/vegafusion-core/src/spec/visitors.rs b/vegafusion-core/src/spec/visitors.rs index 98847464f..bf268a5b5 100644 --- a/vegafusion-core/src/spec/visitors.rs +++ b/vegafusion-core/src/spec/visitors.rs @@ -351,9 +351,9 @@ impl<'a> ChartVisitor for UpdateVarsChartVisitor<'a> { fn visit_signal(&mut self, signal: &SignalSpec, scope: &[u32]) -> Result<()> { // Signal is an update variable if it's not an empty stub - if signal.value.as_option().is_some() - || signal.init.is_some() + if signal.init.is_some() || signal.update.is_some() + || signal.bind.is_some() || !signal.on.is_empty() { self.update_vars diff --git a/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.comm_plan.json b/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.comm_plan.json new file mode 100644 index 000000000..4ea3ee518 --- /dev/null +++ b/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.comm_plan.json @@ -0,0 +1,36 @@ +{ + "server_to_client": [ + { + "namespace": "signal", + "name": "layer_1_bin_maxbins_10_IMDB_Rating_bins", + "scope": [] + }, + { + "namespace": "signal", + "name": "layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins", + "scope": [] + }, + { + "namespace": "data", + "name": "data_0", + "scope": [] + }, + { + "namespace": "data", + "name": "data_1", + "scope": [] + }, + { + "namespace": "data", + "name": "empty", + "scope": [] + } + ], + "client_to_server": [ + { + "namespace": "signal", + "name": "param_5", + "scope": [] + } + ] +} \ No newline at end of file diff --git a/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.updates.json b/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.updates.json new file mode 100644 index 000000000..c501360f1 --- /dev/null +++ b/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.updates.json @@ -0,0 +1,18 @@ +[ + [ + { + "namespace": "signal", + "name": "param_5", + "scope": [], + "value": 3.5 + } + ], + [ + { + "namespace": "signal", + "name": "param_5", + "scope": [], + "value": 6.3 + } + ] +] \ No newline at end of file diff --git a/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.vg.json b/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.vg.json new file mode 100644 index 000000000..5b598829a --- /dev/null +++ b/vegafusion-runtime/tests/specs/custom/aggregate_with_threshold.vg.json @@ -0,0 +1,267 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "background": "white", + "padding": 5, + "width": 300, + "height": 300, + "style": "cell", + "data": [ + { + "name": "source_0", + "url": "https://cdn.jsdelivr.net/npm/vega-datasets@v1.29.0/data/movies.json", + "format": {"type": "json"} + }, + {"name": "empty", "values": [{}]}, + { + "name": "data_0", + "source": "source_0", + "transform": [ + {"type": "filter", "expr": "(datum['IMDB_Rating'] >= param_5)"}, + { + "type": "filter", + "expr": "isValid(datum[\"IMDB_Rating\"]) && isFinite(+datum[\"IMDB_Rating\"]) && isValid(datum[\"Rotten_Tomatoes_Rating\"]) && isFinite(+datum[\"Rotten_Tomatoes_Rating\"])" + } + ] + }, + { + "name": "data_1", + "source": "source_0", + "transform": [ + { + "type": "extent", + "field": "IMDB_Rating", + "signal": "layer_1_bin_maxbins_10_IMDB_Rating_extent" + }, + { + "type": "bin", + "field": "IMDB_Rating", + "as": [ + "bin_maxbins_10_IMDB_Rating", + "bin_maxbins_10_IMDB_Rating_end" + ], + "signal": "layer_1_bin_maxbins_10_IMDB_Rating_bins", + "extent": {"signal": "layer_1_bin_maxbins_10_IMDB_Rating_extent"}, + "maxbins": 10 + }, + { + "type": "extent", + "field": "Rotten_Tomatoes_Rating", + "signal": "layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_extent" + }, + { + "type": "bin", + "field": "Rotten_Tomatoes_Rating", + "as": [ + "bin_maxbins_10_Rotten_Tomatoes_Rating", + "bin_maxbins_10_Rotten_Tomatoes_Rating_end" + ], + "signal": "layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins", + "extent": { + "signal": "layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_extent" + }, + "maxbins": 10 + }, + {"type": "filter", "expr": "(datum['IMDB_Rating'] < param_5)"}, + { + "type": "aggregate", + "groupby": [ + "bin_maxbins_10_IMDB_Rating", + "bin_maxbins_10_IMDB_Rating_end", + "bin_maxbins_10_Rotten_Tomatoes_Rating", + "bin_maxbins_10_Rotten_Tomatoes_Rating_end" + ], + "ops": ["count"], + "fields": [null], + "as": ["__count"] + }, + { + "type": "filter", + "expr": "isValid(datum[\"bin_maxbins_10_IMDB_Rating\"]) && isFinite(+datum[\"bin_maxbins_10_IMDB_Rating\"]) && isValid(datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating\"]) && isFinite(+datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating\"])" + } + ] + } + ], + "signals": [ + { + "name": "param_5", + "value": 5, + "bind": { + "input": "range", + "max": 10, + "min": 0, + "name": "threshold", + "step": 0.1 + } + } + ], + "marks": [ + { + "name": "layer_0_marks", + "type": "symbol", + "style": ["circle"], + "from": {"data": "data_0"}, + "encode": { + "update": { + "opacity": {"value": 0.7}, + "fill": {"value": "#4c78a8"}, + "ariaRoleDescription": {"value": "circle"}, + "description": { + "signal": "\"IMDB Rating: \" + (format(datum[\"IMDB_Rating\"], \"\")) + \"; Rotten Tomatoes Rating: \" + (format(datum[\"Rotten_Tomatoes_Rating\"], \"\"))" + }, + "x": {"scale": "x", "field": "IMDB_Rating"}, + "y": {"scale": "y", "field": "Rotten_Tomatoes_Rating"}, + "shape": {"value": "circle"} + } + } + }, + { + "name": "layer_1_marks", + "type": "symbol", + "style": ["circle"], + "from": {"data": "data_1"}, + "encode": { + "update": { + "fill": {"value": "#4c78a8"}, + "ariaRoleDescription": {"value": "circle"}, + "description": { + "signal": "\"IMDB_Rating (binned): \" + (!isValid(datum[\"bin_maxbins_10_IMDB_Rating\"]) || !isFinite(+datum[\"bin_maxbins_10_IMDB_Rating\"]) ? \"null\" : format(datum[\"bin_maxbins_10_IMDB_Rating\"], \"\") + \" – \" + format(datum[\"bin_maxbins_10_IMDB_Rating_end\"], \"\")) + \"; Rotten_Tomatoes_Rating (binned): \" + (!isValid(datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating\"]) || !isFinite(+datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating\"]) ? \"null\" : format(datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating\"], \"\") + \" – \" + format(datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating_end\"], \"\")) + \"; Count of Records: \" + (format(datum[\"__count\"], \"\"))" + }, + "x": { + "signal": "scale(\"x\", 0.5 * datum[\"bin_maxbins_10_IMDB_Rating\"] + 0.5 * datum[\"bin_maxbins_10_IMDB_Rating_end\"])" + }, + "y": { + "signal": "scale(\"y\", 0.5 * datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating\"] + 0.5 * datum[\"bin_maxbins_10_Rotten_Tomatoes_Rating_end\"])" + }, + "size": {"scale": "size", "field": "__count"}, + "shape": {"value": "circle"} + } + } + }, + { + "name": "layer_2_marks", + "type": "rule", + "style": ["rule"], + "from": {"data": "empty"}, + "encode": { + "update": { + "stroke": {"value": "gray"}, + "strokeWidth": {"value": 6}, + "x": {"scale": "x", "signal": "param_5"}, + "y": {"value": 0}, + "y2": {"field": {"group": "height"}} + } + } + } + ], + "scales": [ + { + "name": "x", + "type": "linear", + "domain": { + "fields": [ + {"data": "data_0", "field": "IMDB_Rating"}, + { + "signal": "[layer_1_bin_maxbins_10_IMDB_Rating_bins.start, layer_1_bin_maxbins_10_IMDB_Rating_bins.stop]" + }, + [{"expr": "param_5"}] + ] + }, + "range": [0, {"signal": "width"}], + "bins": {"signal": "layer_1_bin_maxbins_10_IMDB_Rating_bins"}, + "nice": true, + "zero": true + }, + { + "name": "y", + "type": "linear", + "domain": { + "fields": [ + {"data": "data_0", "field": "Rotten_Tomatoes_Rating"}, + { + "signal": "[layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins.start, layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins.stop]" + } + ] + }, + "range": [{"signal": "height"}, 0], + "bins": {"signal": "layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins"}, + "nice": true, + "zero": true + }, + { + "name": "size", + "type": "linear", + "domain": [0, 160], + "range": [ + 0, + { + "signal": "pow(0.95 * min(width / ((layer_1_bin_maxbins_10_IMDB_Rating_bins.stop - layer_1_bin_maxbins_10_IMDB_Rating_bins.start) / layer_1_bin_maxbins_10_IMDB_Rating_bins.step), height / ((layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins.stop - layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins.start) / layer_1_bin_maxbins_10_Rotten_Tomatoes_Rating_bins.step)), 2)" + } + ], + "zero": true + } + ], + "axes": [ + { + "scale": "x", + "orient": "bottom", + "gridScale": "y", + "grid": true, + "tickCount": {"signal": "ceil(width/40)"}, + "domain": false, + "labels": false, + "aria": false, + "maxExtent": 0, + "minExtent": 0, + "ticks": false, + "zindex": 0 + }, + { + "scale": "y", + "orient": "left", + "gridScale": "x", + "grid": true, + "tickCount": {"signal": "ceil(height/40)"}, + "domain": false, + "labels": false, + "aria": false, + "maxExtent": 0, + "minExtent": 0, + "ticks": false, + "zindex": 0 + }, + { + "scale": "x", + "orient": "bottom", + "grid": false, + "title": "IMDB Rating", + "labelFlush": true, + "labelOverlap": true, + "tickCount": {"signal": "ceil(width/40)"}, + "zindex": 0 + }, + { + "scale": "y", + "orient": "left", + "grid": false, + "title": "Rotten Tomatoes Rating", + "labelOverlap": true, + "tickCount": {"signal": "ceil(height/40)"}, + "zindex": 0 + } + ], + "legends": [ + { + "size": "size", + "symbolType": "circle", + "title": "Count of Records", + "encode": { + "symbols": { + "update": { + "fill": {"value": "#4c78a8"}, + "stroke": {"value": "transparent"} + } + } + } + } + ] +} \ No newline at end of file diff --git a/vegafusion-runtime/tests/test_image_comparison.rs b/vegafusion-runtime/tests/test_image_comparison.rs index 21196fffc..ba436cb6d 100644 --- a/vegafusion-runtime/tests/test_image_comparison.rs +++ b/vegafusion-runtime/tests/test_image_comparison.rs @@ -146,7 +146,8 @@ mod test_custom_specs { case("custom/facet_grouped_bar_with_error_bars_with_sort", 0.001, true), case("custom/binned_ordinal", 0.001, true), case("custom/timeOffset_stocks", 0.001, true), - case("custom/quakes_initial_selection", 0.001, true) + case("custom/quakes_initial_selection", 0.001, true), + case("custom/aggregate_with_threshold", 0.001, true) )] fn test_image_comparison(spec_name: &str, tolerance: f64, extract_inline_values: bool) { println!("spec_name: {spec_name}"); From f470098e503a6507fcea3f4eea6bdfebcc4cc5ee Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sat, 16 Dec 2023 11:18:31 -0500 Subject: [PATCH 2/2] Update expected comm plans --- .../tests/specs/vega/jobs.comm_plan.json | 5 ----- .../tests/specs/vega/layout-wrap.comm_plan.json | 8 +++++++- .../tests/specs/vega/population.comm_plan.json | 8 +++++++- .../tests/specs/vega/population.updates.json | 10 ++++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 vegafusion-runtime/tests/specs/vega/population.updates.json diff --git a/vegafusion-runtime/tests/specs/vega/jobs.comm_plan.json b/vegafusion-runtime/tests/specs/vega/jobs.comm_plan.json index d1e4adc4c..3e56c9969 100644 --- a/vegafusion-runtime/tests/specs/vega/jobs.comm_plan.json +++ b/vegafusion-runtime/tests/specs/vega/jobs.comm_plan.json @@ -5,11 +5,6 @@ "name": "query", "scope": [] }, - { - "namespace": "signal", - "name": "sex", - "scope": [] - }, { "namespace": "data", "name": "_server_jobs", diff --git a/vegafusion-runtime/tests/specs/vega/layout-wrap.comm_plan.json b/vegafusion-runtime/tests/specs/vega/layout-wrap.comm_plan.json index 37a8d7312..fc3e75de6 100644 --- a/vegafusion-runtime/tests/specs/vega/layout-wrap.comm_plan.json +++ b/vegafusion-runtime/tests/specs/vega/layout-wrap.comm_plan.json @@ -26,5 +26,11 @@ "scope": [] } ], - "client_to_server": [] + "client_to_server": [ + { + "namespace": "signal", + "name": "columns", + "scope": [] + } + ] } \ No newline at end of file diff --git a/vegafusion-runtime/tests/specs/vega/population.comm_plan.json b/vegafusion-runtime/tests/specs/vega/population.comm_plan.json index 903c29f37..7d4a08960 100644 --- a/vegafusion-runtime/tests/specs/vega/population.comm_plan.json +++ b/vegafusion-runtime/tests/specs/vega/population.comm_plan.json @@ -31,5 +31,11 @@ "scope": [] } ], - "client_to_server": [] + "client_to_server": [ + { + "namespace": "signal", + "name": "year", + "scope": [] + } + ] } \ No newline at end of file diff --git a/vegafusion-runtime/tests/specs/vega/population.updates.json b/vegafusion-runtime/tests/specs/vega/population.updates.json new file mode 100644 index 000000000..7db777fa6 --- /dev/null +++ b/vegafusion-runtime/tests/specs/vega/population.updates.json @@ -0,0 +1,10 @@ +[ + [ + { + "namespace": "signal", + "name": "year", + "scope": [], + "value": 1960 + } + ] +] \ No newline at end of file