From 6029fb0e7a0c55be6cbbc6bea480fe11924bb29a Mon Sep 17 00:00:00 2001 From: Andrew Paley Date: Wed, 15 Jun 2022 02:44:14 -0500 Subject: [PATCH] feat: addition of generate and format methods on ResponseManager --- devhelper/helper.js | 1198 +++++++++++++++++++++++++++++++++++++++++- devhelper/index.html | 7 + src/index.js | 250 +++++++-- 3 files changed, 1394 insertions(+), 61 deletions(-) diff --git a/devhelper/helper.js b/devhelper/helper.js index 378eb9f..44eb77f 100644 --- a/devhelper/helper.js +++ b/devhelper/helper.js @@ -1,15 +1,1190 @@ import { Satyrn } from "../src/index.js"; -const sample = {"targetEntities": ["Contribution", "Contributor"], "defaultEntity": "Contribution", "filters": [["amount", {"autocomplete": true, "type": "float", "allowMultiple": false, "nicename": "Contribution Amount", "desc": null}], ["inState", {"autocomplete": true, "type": "boolean", "allowMultiple": false, "nicename": "In State Contribution Status", "desc": null}], ["electionYear", {"autocomplete": false, "type": "date:year", "allowMultiple": false, "nicename": "Election Year", "desc": "The year of this election."}], ["contributionRecipient", {"autocomplete": true, "type": "string", "allowMultiple": false, "nicename": "Recipient", "desc": null}], ["contributionDate", {"autocomplete": true, "type": "datetime", "allowMultiple": false, "nicename": "Contribution Date", "desc": null}]], "columns": [{"key": "amount", "nicename": "Contribution Amount", "width": "20.0%", "sortable": true}, {"key": "inState", "nicename": "In State Contribution Status", "width": "20.0%", "sortable": true}, {"key": "electionYear", "nicename": "Election Year", "width": "20.0%", "sortable": true}, {"key": "contributionRecipient", "nicename": "Recipient", "width": "20.0%", "sortable": true}, {"key": "contributionDate", "nicename": "Contribution Date", "width": "20.0%", "sortable": true}], "defaultSort": {"key": "amount", "direction": "desc"}, "fieldUnits": {}, +const sampleRingFromAPI = { + "targetEntities": [ + "Contribution", + "Contributor" + ], + "defaultEntity": "Contribution", + "filters": [ + [ + "amount", + { + "autocomplete": true, + "type": "float", + "allowMultiple": false, + "nicename": "Contribution Amount", + "desc": null + } + ], + [ + "inState", + { + "autocomplete": true, + "type": "boolean", + "allowMultiple": false, + "nicename": "In State Contribution Status", + "desc": null + } + ], + [ + "electionYear", + { + "autocomplete": true, + "type": "date:year", + "allowMultiple": false, + "nicename": "Election Year", + "desc": "The year of this election." + } + ], + [ + "contributionRecipient", + { + "autocomplete": true, + "type": "string", + "allowMultiple": false, + "nicename": "Recipient", + "desc": null + } + ], + [ + "contributionDate", + { + "autocomplete": true, + "type": "date", + "allowMultiple": false, + "nicename": "Contribution Date", + "desc": null + } + ] + ], + "columns": [ + { + "key": "amount", + "nicename": "Contribution Amount", + "width": "20.0%", + "sortable": true + }, + { + "key": "inState", + "nicename": "In State Contribution Status", + "width": "20.0%", + "sortable": true + }, + { + "key": "electionYear", + "nicename": "Election Year", + "width": "20.0%", + "sortable": true + }, + { + "key": "contributionRecipient", + "nicename": "Recipient", + "width": "20.0%", + "sortable": true + }, + { + "key": "contributionDate", + "nicename": "Contribution Date", + "width": "20.0%", + "sortable": true + } + ], + "defaultSort": { + "key": "amount", + "direction": "desc" + }, + "fieldUnits": {}, + "analysisSpace": { + "_self": { + "entity": "Contribution", + "nicename": [ + "contribution", + "contributions" + ], + "relType": "o2o", + "attributes": [ + { + "type": "float", + "nicename": [ + "Contribution Amount", + "Contribution Amounts" + ], + "unit": [ + "dollar", + "dollars" + ], + "targetField": "amount" + }, + { + "type": "boolean", + "nicename": [ + "In State Contribution Status", + "In State Contribution Statuses" + ], + "unit": null, + "targetField": "inState" + }, + { + "type": "date:year", + "nicename": [ + "Election Year", + "Election Years" + ], + "unit": null, + "targetField": "electionYear" + }, + { + "type": "string", + "nicename": [ + "Recipient", + "Recipients" + ], + "unit": null, + "targetField": "contributionRecipient" + }, + { + "type": "date", + "nicename": [ + "Contribution Date", + "Contribution Dates" + ], + "unit": null, + "targetField": "contributionDate" + }, + { + "type": "id", + "nicename": false, + "unit": "Contribution", + "targetField": "id" + } + ] + }, + "ContribToContributor": { + "entity": "Contributor", + "nicename": [ + "contributor", + "contributors" + ], + "relType": "m2o", + "attributes": [ + { + "type": "string", + "nicename": [ + "Contributor", + "Contributor" + ], + "unit": null, + "targetField": "name" + }, + { + "type": "string", + "nicename": [ + "Contributor Parent Organization", + "Contributor Parent Organizations" + ], + "unit": null, + "targetField": "parentOrg" + }, + { + "type": "string", + "nicename": [ + "Contributor Sector", + "Contributor Sectors" + ], + "unit": null, + "targetField": "area" + }, + { + "type": "id", + "nicename": false, + "unit": "Contributor", + "targetField": "id" + } + ] + } + }, + "includesRenderer": false, + "targetModelName": "Contribution", + "operations": { + "average": { + "required": { + "target": { + "validInputs": [ + "integer", + "float" + ], + "fieldType": "target" + } + }, + "template": "Average {target}", + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "count": { + "required": { + "target": { + "validInputs": [ + "id" + ], + "fieldType": "target" + } + }, + "template": "Count of unique {target}", + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "sum": { + "required": { + "target": { + "validInputs": [ + "integer", + "float" + ], + "fieldType": "target" + } + }, + "template": "Total {target}", + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "min": { + "required": { + "target": { + "validInputs": [ + "integer", + "float" + ], + "fieldType": "target" + } + }, + "template": "Min of {target}", + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "max": { + "required": { + "target": { + "validInputs": [ + "integer", + "float" + ], + "fieldType": "target" + } + }, + "template": "Max of {target}", + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "median": { + "required": { + "target": { + "validInputs": [ + "integer", + "float" + ], + "fieldType": "target" + } + }, + "template": "Median {target}", + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "averageCount": { + "required": { + "target": { + "validInputs": [ + "id" + ], + "fieldType": "target" + }, + "per": { + "validInputs": [ + "id" + ], + "fieldType": "group" + } + }, + "template": "Average count of {target} per {per}", + "units": "target/per", + "type": "recursive", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "averageSum": { + "required": { + "target": { + "validInputs": [ + "float", + "integer" + ], + "fieldType": "target" + }, + "per": { + "validInputs": [ + "id" + ], + "fieldType": "group" + } + }, + "template": "Average sum of {target} per {per}", + "units": "target/per", + "type": "recursive", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "percentage": { + "required": { + "target": { + "validInputs": [ + "string", + "boolean" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "boolean", + "string" + ], + "options": "any", + "allowMultiple": true + } + ] + } + }, + "template": "Percentage of {target}", + "units": "percentage", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "oneHot": { + "required": { + "target": { + "validInputs": [ + "string", + "boolean" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "boolean", + "string" + ], + "options": "any", + "allowMultiple": true + } + ] + } + }, + "units": "none", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "None": { + "required": { + "target": { + "validInputs": [ + "integer", + "float", + "boolean", + "string" + ], + "fieldType": "target" + } + }, + "units": "unchanged", + "type": "simple", + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + } + }, + "distribution": { + "required": { + "target": { + "validInputs": [ + "integer", + "float", + "average", + "count" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "integer", + "float" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + } + ] + }, + "over": { + "fieldType": "group", + "validInputs": [ + "id", + "boolean", + "string" + ], + "parameters": null + } + }, + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + }, + "template": "Distribution of {target} over {over}", + "type": "complex" + }, + "comparison": { + "required": { + "target1": { + "validInputs": [ + "string", + "boolean", + "integer", + "float", + "id" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "boolean", + "string" + ], + "options": "any", + "allowMultiple": false + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "int", + "float" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "id" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + } + ] + }, + "target2": { + "validInputs": [ + "string", + "boolean", + "integer", + "float", + "id" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "boolean", + "string" + ], + "options": "any", + "allowMultiple": true + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "int", + "float" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "id" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + } + ] + }, + "group": { + "internalId": "group", + "fieldType": "group", + "validInputs": [ + "id", + "boolean" + ], + "parameters": null + } + }, + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + }, + "template": "Comparison between {group}'s {target1} and {target2}", + "type": "complex" + }, + "correlation": { + "required": { + "target1": { + "validInputs": [ + "string", + "boolean", + "integer", + "float", + "id" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "boolean", + "string" + ], + "options": "any", + "allowMultiple": false + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "integer", + "float" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "id" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + } + ] + }, + "target2": { + "validInputs": [ + "string", + "boolean", + "integer", + "float", + "id" + ], + "fieldType": "target", + "parameters": [ + { + "question": "language to be asked goes here", + "inputTypes": [ + "boolean", + "string" + ], + "options": "any", + "allowMultiple": true + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "int", + "float" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + }, + { + "question": "language to be asked goes here", + "inputTypes": [ + "id" + ], + "options": "aggregation", + "required": false, + "allowMultiple": false + } + ] + }, + "group": { + "internalId": "group", + "fieldType": "group", + "validInputs": [ + "id", + "boolean" + ], + "parameters": null + } + }, + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + }, + "template": "Correlation between {group}'s {target1} and {target2}", + "type": "complex" + }, + "summaryStatistics": { + "required": { + "target": { + "validInputs": [ + "integer", + "float" + ], + "fieldType": "target", + "parameters": [] + } + }, + "optional": { + "groupBy": { + "allowed": true, + "maxDepth": 2, + "validInputs": [ + "id", + "integer", + "float", + "string", + "boolean" + ], + "parameters": [ + { + "inputTypes": [ + "integer", + "float" + ], + "options": [ + "percentile", + "threshold" + ], + "allowMultiple": false + } + ] + }, + "timeSeries": { + "allowed": true, + "maxDepth": 1, + "validInputs": [ + "date", + "datetime", + "date:year" + ] + } + }, + "spawned": { + "target0": { + "spawnOf": "target", + "fieldType": "target" + }, + "target1": { + "spawnOf": "target", + "fieldType": "target" + }, + "target2": { + "spawnOf": "target", + "fieldType": "target" + }, + "target3": { + "spawnOf": "target", + "fieldType": "target" + } + }, + "template": "Summary Statistics of {target}", + "type": "complex" + } + } +} -"analysisSpace": {"_self": {"entity": "Contribution", "nicename": ["Contribution", "Contribution"], "relType": "o2o", "attributes": [{"type": "float", "nicename": ["Contribution Amount", "Contribution Amounts"], "unit": ["dollar", "dollars"], "targetField": "amount"}, {"type": "boolean", "nicename": ["In State Contribution Status", "In State Contribution Statuses"], "unit": null, "targetField": "inState"}, {"type": "date:year", "nicename": ["Election Year", "Election Years"], "unit": null, "targetField": "electionYear"}, {"type": "string", "nicename": ["Recipient", "Recipients"], "unit": null, "targetField": "contributionRecipient"}, {"type": "datetime", "nicename": ["Contribution Date", "Contribution Dates"], "unit": null, "targetField": "contributionDate"}, {"type": "id", "nicename": false, "unit": "Contribution", "targetField": "id"}]}, "ContribToContributor": {"entity": "Contributor", "nicename": ["Contributor", "Contributor"], "relType": "m2o", "attributes": [{"type": "string", "nicename": ["Contributor", "Contributor"], "unit": null, "targetField": "name"}, {"type": "string", "nicename": ["Contributor Parent Organization", "Contributor Parent Organizations"], "unit": null, "targetField": "parentOrg"}, {"type": "string", "nicename": ["Contributor Sector", "Contributor Sectors"], "unit": null, "targetField": "area"}, {"type": "id", "nicename": false, "unit": "Contributor", "targetField": "id"}]}}, - -"includesRenderer": false, "targetModelName": "Contribution", "operations": {"average": {"required": {"target": {"validInputs": ["integer", "float"], "fieldType": "target"}}, "template": "Average {target}", "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "count": {"required": {"target": {"validInputs": ["id"], "fieldType": "target"}}, "template": "Count of unique {target}", "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "sum": {"required": {"target": {"validInputs": ["integer", "float"], "fieldType": "target"}}, "template": "Total {target}", "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "min": {"required": {"target": {"validInputs": ["integer", "float"], "fieldType": "target"}}, "template": "Min of {target}", "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "max": {"required": {"target": {"validInputs": ["integer", "float"], "fieldType": "target"}}, "template": "Max of {target}", "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "median": {"required": {"target": {"validInputs": ["integer", "float"], "fieldType": "target"}}, "template": "Median {target}", "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "averageCount": {"required": {"target": {"validInputs": ["id"], "fieldType": "target"}, "per": {"validInputs": ["id"], "fieldType": "group"}}, "template": "Average Count of {target} per {per}", "units": "target/per", "type": "recursive", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "averageSum": {"required": {"target": {"validInputs": ["float", "integer"], "fieldType": "target"}, "per": {"validInputs": ["id"], "fieldType": "group"}}, "template": "Average Sum of {target} per {per}", "units": "target/per", "type": "recursive", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "percentage": {"required": {"target": {"validInputs": ["string", "boolean"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["boolean", "string"], "options": "any", "allowMultiple": true}]}}, "template": "Percentage of {target}", "units": "percentage", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "oneHot": {"required": {"target": {"validInputs": ["string", "boolean"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["boolean", "string"], "options": "any", "allowMultiple": true}]}}, "units": "none", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "None": {"required": {"target": {"validInputs": ["integer", "float", "boolean", "string"], "fieldType": "target"}}, "units": "unchanged", "type": "simple", "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}}, "summaryStatistics": {"required": {"target": {"validInputs": ["integer", "float"], "fieldType": "target", "parameters": []}}, "optional": {"groupBy": {"allowed": true, "maxDepth": 2, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}, "spawned": {"target0": {"spawnOf": "target", "fieldType": "target"}, "target1": {"spawnOf": "target", "fieldType": "target"}, "target2": {"spawnOf": "target", "fieldType": "target"}, "target3": {"spawnOf": "target", "fieldType": "target"}}, "template": "Summary Statistics of {target}", "type": "complex"}, "correlation": {"required": {"target1": {"validInputs": ["string", "boolean", "integer", "float", "id"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["boolean", "string"], "options": "any", "allowMultiple": false}, {"question": "language to be asked goes here", "inputTypes": ["integer", "float"], "options": "aggregation", "required": false, "allowMultiple": false}, {"question": "language to be asked goes here", "inputTypes": ["id"], "options": "aggregation", "required": false, "allowMultiple": false}]}, "target2": {"validInputs": ["string", "boolean", "integer", "float", "id"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["boolean", "string"], "options": "any", "allowMultiple": true}, {"question": "language to be asked goes here", "inputTypes": ["int", "float"], "options": "aggregation", "required": false, "allowMultiple": false}, {"question": "language to be asked goes here", "inputTypes": ["id"], "options": "aggregation", "required": false, "allowMultiple": false}]}, "group": {"internalId": "group", "fieldType": "group", "validInputs": ["id", "boolean"], "parameters": null}}, "optional": {"groupBy": {"allowed": true, "maxDepth": 1, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}, "template": "Correlation between {group}'s {target1} and {target2}", "type": "complex"}, "comparison": {"required": {"target1": {"validInputs": ["string", "boolean", "integer", "float", "id"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["boolean", "string"], "options": "any", "allowMultiple": false}, {"question": "language to be asked goes here", "inputTypes": ["int", "float"], "options": "aggregation", "required": false, "allowMultiple": false}, {"question": "language to be asked goes here", "inputTypes": ["id"], "options": "aggregation", "required": false, "allowMultiple": false}]}, "target2": {"validInputs": ["string", "boolean", "integer", "float", "id"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["boolean", "string"], "options": "any", "allowMultiple": true}, {"question": "language to be asked goes here", "inputTypes": ["int", "float"], "options": "aggregation", "required": false, "allowMultiple": false}, {"question": "language to be asked goes here", "inputTypes": ["id"], "options": "aggregation", "required": false, "allowMultiple": false}]}, "group": {"internalId": "group", "fieldType": "group", "validInputs": ["id", "boolean"], "parameters": null}}, "optional": {"groupBy": {"allowed": true, "maxDepth": 1, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}, "template": "Comparison between {group}'s {target1} and {target2}", "type": "complex"}, "distribution": {"required": {"target": {"validInputs": ["integer", "float", "average", "count"], "fieldType": "target", "parameters": [{"question": "language to be asked goes here", "inputTypes": ["integer", "float"], "options": "aggregation", "required": false, "allowMultiple": false}]}, "over": {"fieldType": "group", "validInputs": ["id", "boolean", "string"], "parameters": null}}, "optional": {"groupBy": {"allowed": true, "maxDepth": 1, "validInputs": ["id", "integer", "float", "string", "boolean"], "parameters": [{"inputTypes": ["integer", "float"], "options": ["percentile", "threshold"], "allowMultiple": false}]}, "timeSeries": {"allowed": true, "maxDepth": 1, "validInputs": ["date", "datetime", "date:year"]}}, "template": "Distribution of {target} over {over}", "type": "complex"}}} +const sampleAnalysisOutputs = {"test_averagecount_contribution_contributor": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "id"}, "per": {"entity": "Contributor", "field": "id"}, "op": "averageCount", "relationships": ["ContribToContributor"]}, "expected_results": {"counts": {"Contribution//id": 200, "Contributor//id": 118}, "fieldNames": [{"entity": "Contribution", "field": "id", "op": "averageCount", "per": {"entity": "Contributor", "field": "id"}}], "length": 1, "results": [[1.69]], "units": {"results": ["Contribution/Contributor"]}}}, "test_averagecount_contribution_contributor_groupby_area": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "id"}, "per": {"entity": "Contributor", "field": "id"}, "op": "averageCount", "groupBy": [{"entity": "Contributor", "field": "area"}], "relationships": ["ContribToContributor"]}, "expected_results": {"counts": {"Contribution//id": 200, "Contributor//id": 118}, "fieldNames": [{"entity": "Contributor", "field": "area"}, {"entity": "Contribution", "field": "id", "op": "averageCount", "per": {"entity": "Contributor", "field": "id"}}], "length": 17, "results": [["Air transport unions", 1.0], ["Automotive unions", 4.0], ["Communications & hi-tech unions", 1.0], ["Construction unions", 1.55], ["Electrical workers/IBEW", 2.2], ["Entertainment unions", 1.75], ["Fire fighters unions and associations", 2.0], ["Health worker unions", 3.5], ["Labor unions", 2.2], ["Manufacturing unions", 1.0], ["Other unions", 1.5], ["Police unions & associations", 2.0], ["Railroad unions", 1.5], ["Retail trade unions", 2.33], ["State & local government employee unions", 1.33], ["Teachers unions", 1.83], ["Teamsters unions", 1.14]], "units": {"results": ["Contributor Sector", "Contribution/Contributor"]}}}, "test_count_contribution": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "id"}, "op": "count", "relationships": []}, "expected_results": {"counts": {"Contribution//id": 200}, "fieldNames": [{"entity": "Contribution", "field": "id", "op": "count"}], "length": 1, "results": [[200]], "units": {"results": ["Contribution"]}}}, "test_count_contributor_groupby_area": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contributor", "field": "id"}, "op": "count", "groupBy": [{"entity": "Contributor", "field": "area"}], "relationships": ["ContribToContributor"]}, "expected_results": {"counts": {"Contributor//id": 118}, "fieldNames": [{"entity": "Contributor", "field": "area"}, {"entity": "Contributor", "field": "id", "op": "count"}], "length": 17, "results": [["Air transport unions", 1], ["Automotive unions", 2], ["Communications & hi-tech unions", 1], ["Construction unions", 53], ["Electrical workers/IBEW", 10], ["Entertainment unions", 4], ["Fire fighters unions and associations", 2], ["Health worker unions", 2], ["Labor unions", 5], ["Manufacturing unions", 1], ["Other unions", 4], ["Police unions & associations", 3], ["Railroad unions", 4], ["Retail trade unions", 3], ["State & local government employee unions", 3], ["Teachers unions", 6], ["Teamsters unions", 14]], "units": {"results": ["Contributor Sector", "Contributor"]}}}, "test_count_contributor_groupby_parentorg": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contributor", "field": "id"}, "op": "count", "groupBy": [{"entity": "Contributor", "field": "parentOrg"}], "relationships": []}, "expected_results": {"counts": {"Contributor//id": 118}, "fieldNames": [{"entity": "Contributor", "field": "parentOrg"}, {"entity": "Contributor", "field": "id", "op": "count"}], "length": 28, "results": [["AMERICAN FEDERATION OF TEACHERS / AFT", 3], ["BROTHERHOOD OF LOCOMOTIVE ENGINEERS & TRAINMEN / BLET", 1], ["COMMUNICATIONS WORKERS OF AMERICA / CWA", 1], ["FRATERNAL ORDER OF POLICE ASSOCIATES / FOP", 1], ["ILLINOIS AFL-CIO", 1], ["INTERNATIONAL ALLIANCE OF THEATRICAL STAGE EMPLOYEES / IATSE", 3], ["INTERNATIONAL ASSOCIATION OF BRIDGE STRUCTURAL ORNAMENTAL & REINFORCING IRON WORKERS", 5], ["INTERNATIONAL ASSOCIATION OF FIRE FIGHTERS / IAFF", 2], ["INTERNATIONAL ASSOCIATION OF HEAT & FROST INSULATORS & ALLIED WORKERS / HFIAW", 1], ["INTERNATIONAL ASSOCIATION OF SHEET METAL AIR RAIL & TRANSPORTATION WORKERS / SMART", 2], ["INTERNATIONAL BROTHERHOOD OF BOILERMAKERS IRON SHIP BUILDERS BLACKSMITHS FORGERS & HELPERS / IBB", 2], ["INTERNATIONAL BROTHERHOOD OF ELECTRICAL WORKERS / IBEW", 9], ["INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 13], ["INTERNATIONAL UNION OF OPERATING ENGINEERS / IUOE", 6], ["INTERNATIONAL UNION OF PAINTERS & ALLIED TRADES / IUPAT", 2], ["LABORERS INTERNATIONAL UNION OF NORTH AMERICA / LIUNA", 7], ["NATIONAL AFL-CIO", 5], ["NATIONAL EDUCATION ASSOCIATION / NEA", 1], ["No value", 25], ["SERVICE EMPLOYEES INTERNATIONAL UNION / SEIU", 3], ["SHEET METAL WORKERS INTERNATIONAL ASSOCIATION / SMWIA", 1], ["UNITE HERE! INTERNATIONAL UNION", 1], ["UNITED ASSOCIATION OF JOURNEYMEN & APPRENTICES OF THE PLUMBING & PIPE FITTING INDUSTRY OF THE UNITED STATES & CANADA / UA", 10], ["UNITED AUTOMOBILE AEROSPACE & AGRICULTURAL IMPLEMENT WORKERS OF AMERICA / UAW", 2], ["UNITED BROTHERHOOD OF CARPENTERS & JOINERS / UBC", 7], ["UNITED FOOD & COMMERCIAL WORKERS INTERNATIONAL UNION / UFCW", 2], ["UNITED STEEL PAPER & FORESTRY RUBBER MANUFACTURING ENERGY ALLIED INDUSTRIAL & SERVICE WORKERS INTERNATIONAL / USW", 1], ["UNITED UNION OF ROOFERS WATERPROOFERS & ALLIED WORKERS", 1]], "units": {"results": ["Contributor Parent Organization", "Contributor"]}}}, "test_filter_area_average_amount": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "amount"}, "op": "average", "relationships": ["ContribToContributor"], "query": {"AND": [[{"entity": "Contributor", "field": "area"}, "Other unions", "exact"]]}}, "expected_results": {"counts": {"Contribution//id": 6}, "fieldNames": [{"entity": "Contribution", "field": "amount", "op": "average"}], "length": 1, "results": [[1500.0]], "units": {"results": ["dollar"]}}}, "test_filter_area_average_amount_groupby_instate_time_year": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "amount"}, "op": "average", "timeSeries": {"entity": "Contribution", "field": "electionYear"}, "groupBy": [{"entity": "Contribution", "field": "inState"}], "relationships": ["ContribToContributor"], "query": {"AND": [[{"entity": "Contributor", "field": "area"}, "Labor unions", "contains"]]}}, "expected_results": {"counts": {"Contribution//id": 11}, "fieldNames": [{"entity": "Contribution", "field": "inState"}, {"entity": "Contribution", "field": "electionYear"}, {"entity": "Contribution", "field": "amount", "op": "average"}], "length": 2, "results": [[true, 2010.0, 7406.25], [true, 2014.0, 3600.0]], "units": {"results": ["In State Contribution Status", "Election Year", "dollar"]}}}, "test_filter_area_sum_amount_time_year": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "amount"}, "op": "sum", "timeSeries": {"entity": "Contribution", "field": "electionYear"}, "relationships": ["ContribToContributor"], "query": {"AND": [[{"entity": "Contributor", "field": "area"}, "Other unions", "exact"]]}}, "expected_results": {"counts": {"Contribution//id": 6}, "fieldNames": [{"entity": "Contribution", "field": "electionYear"}, {"entity": "Contribution", "field": "amount", "op": "sum"}], "length": 2, "results": [[2010.0, 8000.0], [2014.0, 1000.0]], "units": {"results": ["Election Year", "dollar"]}}}, "test_filter_year_count_contributor": {"search_opts": {"electionYear": "2010"}, "analysis_opts": {"target": {"entity": "Contributor", "field": "id"}, "op": "count", "relationships": ["ContribToContributor"]}, "expected_results": {"counts": {"Contributor//id": 84}, "fieldNames": [{"entity": "Contributor", "field": "id", "op": "count"}], "length": 1, "results": [[84]], "units": {"results": ["Contributor"]}}}, "test_filter_year_min_amount": {"search_opts": {"electionYear": "2010"}, "analysis_opts": {"target": {"entity": "Contribution", "field": "amount"}, "op": "min", "relationships": []}, "expected_results": {"counts": {"Contribution//id": 129}, "fieldNames": [{"entity": "Contribution", "field": "amount", "op": "min"}], "length": 1, "results": [[100.0]], "units": {"results": ["dollar"]}}}, "test_filter_year_min_amount_json": {"search_opts": {}, "analysis_opts": {"target": {"entity": "Contribution", "field": "amount"}, "query": {"AND": [[{"entity": "Contribution", "field": "electionYear"}, 2010, "exact"]]}, "op": "min", "relationships": []}, "expected_results": {"counts": {"Contribution//id": 129}, "fieldNames": [{"entity": "Contribution", "field": "amount", "op": "min"}], "length": 1, "results": [[100.0]], "units": {"results": ["dollar"]}}}, "test_filter_year_sum_amount_groupby_contributor": {"search_opts": {"electionYear": "2014"}, "analysis_opts": {"target": {"entity": "Contribution", "field": "amount"}, "op": "sum", "groupBy": [{"entity": "Contributor", "field": "id"}], "relationships": ["ContribToContributor"]}, "expected_results": {"counts": {"Contribution//id": 66, "Contributor//id": 47}, "fieldNames": [{"entity": "Contributor", "field": "id"}, {"entity": "Contributor", "field": "reference"}, {"entity": "Contribution", "field": "amount", "op": "sum"}], "length": 47, "results": [[1, "INTERNATIONAL BROTHERHOOD OF ELECTRICAL WORKERS / IBEW from NATIONAL AFL-CIO", 36602.3], [6, "INTERNATIONAL ASSOCIATION OF SHEET METAL AIR RAIL & TRANSPORTATION WORKERS / SMART from No value", 1000.0], [8, "ELECTRICAL WORKERS LOCAL 145 from INTERNATIONAL BROTHERHOOD OF ELECTRICAL WORKERS / IBEW", 250.0], [9, "IRONWORKERS LOCAL 63 from INTERNATIONAL ASSOCIATION OF BRIDGE STRUCTURAL ORNAMENTAL & REINFORCING IRON WORKERS", 5000.0], [12, "12TH CONGRESSIONAL DISTRICT OF ILLINOIS AFL-CIO from ILLINOIS AFL-CIO", 5300.0], [13, "FOOD & COMMERCIAL WORKERS LOCAL 881 from UNITED FOOD & COMMERCIAL WORKERS INTERNATIONAL UNION / UFCW", 84200.0], [18, "CHICAGO FRATERNAL ORDER OF POLICE LODGE 7 from FRATERNAL ORDER OF POLICE ASSOCIATES / FOP", 2500.0], [20, "TEAMSTERS LOCAL 705 from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 25000.0], [21, "ILLINOIS AFL-CIO from NATIONAL AFL-CIO", 5000.0], [23, "STATE UNIVERSITIES ANNUITANTS ASSOCIATION from No value", 200.0], [24, "UAW REGION 4 from UNITED AUTOMOBILE AEROSPACE & AGRICULTURAL IMPLEMENT WORKERS OF AMERICA / UAW", 50500.0], [27, "TEAMSTERS LOCAL 777 from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 500.0], [32, "THEATRICAL STAGE EMPLOYEES LOCAL 2 from INTERNATIONAL ALLIANCE OF THEATRICAL STAGE EMPLOYEES / IATSE", 5000.0], [33, "BROTHERHOOD OF LOCOMOTIVE ENGINEERS & TRAINMEN / BLET from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 1000.0], [36, "CHICAGO & COOK COUNTY BUILDING & CONSTRUCTION TRADES COUNCIL from No value", 1000.0], [37, "TEAMSTERS JOINT COUNCIL 25 from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 2500.0], [41, "SHEET METAL WORKERS LOCAL 265 from INTERNATIONAL ASSOCIATION OF SHEET METAL AIR RAIL & TRANSPORTATION WORKERS / SMART", 1000.0], [47, "IRONWORKERS DISTRICT COUNCIL OF CHICAGO & VICINITY from INTERNATIONAL ASSOCIATION OF BRIDGE STRUCTURAL ORNAMENTAL & REINFORCING IRON WORKERS", 20000.0], [48, "BROTHERHOOD OF RAILROAD SIGNALMEN / BRS from No value", 1000.0], [49, "UNITED ASSOCIATION OF JOURNEYMEN & APPRENTICES OF THE PLUMBING & PIPE FITTING INDUSTRY OF THE UNITED STATES & CANADA / UA from No value", 100000.0], [51, "TEAMSTERS LOCAL 50 from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 3500.0], [52, "SEIU HCII from SERVICE EMPLOYEES INTERNATIONAL UNION / SEIU", 750000.0], [58, "SHEET METAL WORKERS LOCAL 268 from INTERNATIONAL ASSOCIATION OF SHEET METAL AIR RAIL & TRANSPORTATION WORKERS / SMART", 500.0], [59, "LABORERS INTERNATIONAL UNION OF NORTH AMERICA / LIUNA from LABORERS INTERNATIONAL UNION OF NORTH AMERICA / LIUNA", 200000.0], [60, "FOOD & COMMERCIAL WORKERS LOCAL 1546 from UNITED FOOD & COMMERCIAL WORKERS INTERNATIONAL UNION / UFCW", 2500.0], [62, "ILLINOIS PIPE TRADES ASSOCIATION from UNITED ASSOCIATION OF JOURNEYMEN & APPRENTICES OF THE PLUMBING & PIPE FITTING INDUSTRY OF THE UNITED STATES & CANADA / UA", 75000.0], [63, "PLUMBERS & PIPEFITTERS LOCAL 99 from UNITED ASSOCIATION OF JOURNEYMEN & APPRENTICES OF THE PLUMBING & PIPE FITTING INDUSTRY OF THE UNITED STATES & CANADA / UA", 5600.0], [64, "CHICAGO & NORTHEASTERN ILLINOIS DISTRICT COUNCIL OF CARPENTERS from UNITED BROTHERHOOD OF CARPENTERS & JOINERS / UBC", 5000.0], [65, "PEORIA FIRE FIGHTERS LOCAL 50 from INTERNATIONAL ASSOCIATION OF FIRE FIGHTERS / IAFF", 2500.0], [66, "ILLINOIS EDUCATION ASSOCIATION from NATIONAL EDUCATION ASSOCIATION / NEA", 1000.0], [69, "UNITED FOOD & COMMERCIAL WORKERS INTERNATIONAL UNION / UFCW from No value", 250000.0], [72, "THEATRICAL STAGE EMPLOYEES LOCAL 476 from INTERNATIONAL ALLIANCE OF THEATRICAL STAGE EMPLOYEES / IATSE", 4000.0], [76, "525 POLITICAL CLUB from No value", 750.0], [78, "DUPAGE COUNTY BUILDING & CONSTRUCTION TRADES COUNCIL from NATIONAL AFL-CIO", 1000.0], [81, "THEATRICAL STAGE EMPLOYEES LOCAL 750 from INTERNATIONAL ALLIANCE OF THEATRICAL STAGE EMPLOYEES / IATSE", 1000.0], [82, "ILLINOIS BROTHERHOOD OF LOCOMOTIVE ENGINEERS & TRAINMEN from BROTHERHOOD OF LOCOMOTIVE ENGINEERS & TRAINMEN / BLET", 250.0], [86, "PLUMBERS & PIPEFITTERS LOCAL 597 from UNITED ASSOCIATION OF JOURNEYMEN & APPRENTICES OF THE PLUMBING & PIPE FITTING INDUSTRY OF THE UNITED STATES & CANADA / UA", 20000.0], [95, "TEAMSTERS LOCAL 727 from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 10000.0], [99, "COMMUNICATIONS WORKERS DISTRICT 4 from COMMUNICATIONS WORKERS OF AMERICA / CWA", 2500.0], [100, "UNITED AUTOMOBILE AEROSPACE & AGRICULTURAL IMPLEMENT WORKERS OF AMERICA / UAW from UNITED AUTOMOBILE AEROSPACE & AGRICULTURAL IMPLEMENT WORKERS OF AMERICA / UAW", 250000.0], [102, "ELECTRICAL WORKERS LOCAL 364 from INTERNATIONAL BROTHERHOOD OF ELECTRICAL WORKERS / IBEW", 5000.0], [103, "ELECTRICAL WORKERS LOCAL 146 from INTERNATIONAL BROTHERHOOD OF ELECTRICAL WORKERS / IBEW", 3750.0], [104, "ELECTRICAL WORKERS LOCAL 34 from INTERNATIONAL BROTHERHOOD OF ELECTRICAL WORKERS / IBEW", 5000.0], [106, "WILL-GRUNDY COUNTIES CENTRAL TRADES & LABOR COUNCIL from No value", 500.0], [108, "CARPENTERS & JOINERS LOCAL 790 from UNITED BROTHERHOOD OF CARPENTERS & JOINERS / UBC", 500.0], [109, "INTERNATIONAL ASSOCIATION OF HEAT & FROST INSULATORS & ALLIED WORKERS / HFIAW from No value", 10600.0], [114, "TEAMSTERS LOCAL 627 from INTERNATIONAL BROTHERHOOD OF TEAMSTERS / IBT", 500.0]], "units": {"results": ["Contributor", "Contributor", "dollar"]}}}} // sample.analysisSpace._self = sample.analysisSpace[null] // delete sample.analysisSpace[null] -const satyrn = new Satyrn(sample.defaultEntity, sample.operations, sample.analysisSpace) +const satyrn = new Satyrn(sampleRingFromAPI.defaultEntity, sampleRingFromAPI.operations, sampleRingFromAPI.analysisSpace) const plans = satyrn.planManager.generate() let planPayload = `

Ring Info

@@ -28,3 +1203,16 @@ plans.forEach(plan => { }) document.getElementById("planview").innerHTML = planPayload + +// now do the analysis +let analysisPayload = `

Sample Analysis Outputs

` + +const analysisTemplate = (tname, ar) => `
  • + TEST_NAME: ${tname}
    SEARCH FILTERS: ${JSON.stringify(ar.search_opts)}
    PLAN: ${JSON.stringify(ar.analysis_opts)}
    RESULTS: ${JSON.stringify(ar.expected_results)}
    RESULTS LANGUAGE: ${satyrn.responseManager.generate(ar.search_opts, ar.analysis_opts, ar.expected_results)}
    RESULTS FORMAT: ${JSON.stringify(satyrn.responseManager.format(ar.analysis_opts, ar.expected_results))} +
    • ` + +Object.entries(sampleAnalysisOutputs).forEach(ao => { + analysisPayload += analysisTemplate(ao[0], ao[1]) +}) + +document.getElementById("analysisView").innerHTML = analysisPayload diff --git a/devhelper/index.html b/devhelper/index.html index 1c15c53..7ce2572 100644 --- a/devhelper/index.html +++ b/devhelper/index.html @@ -2,5 +2,12 @@ +
    +
    + + + diff --git a/src/index.js b/src/index.js index 05aca6f..9ca0dbb 100644 --- a/src/index.js +++ b/src/index.js @@ -1,46 +1,9 @@ // KNOWN TODOS: // - multi-hop relationship chaining // - date granularity protections per-attribute +// - implement distribution, correlation, percentage, and oneHot plan generation and results management (and remove the generateMocks hack) const generateMocks = (primaryRing) => [ - { - statement: "Average count of contributions per contributor", - parameters: [], - plan: { - op: "averageCount", - target: { - entity: "Contribution", - field: "id" - }, - per: { - entity: "Contributor", - field: "id" - }, - rings: [primaryRing], - relationships: ["ContribToContributor"] // how to relate contributions to contributors - } - }, - { - statement: "Average count of contributions per contributor grouped by in-state status", - parameters: [], - plan: { - op: "averageCount", - target: { - entity: "Contribution", - field: "id" - }, - per: { - entity: "Contributor", - field: "id" - }, - groupBy: [{ // now with a group by - entity: "Contribution", - field: "inState" - }], - rings: [primaryRing], - relationships: ["ContribToContributor"] - } - }, { statement: "Distribution of contribution amount across party grouped by in-state status", parameters: [ @@ -151,13 +114,131 @@ class Satyrn { this.analysisSpace = analysisSpace this.primaryRing = primaryRing this.planManager = new PlanManager(targetEntity, operations, analysisSpace, primaryRing) - this.responseManager = new ResponseManager() + this.responseManager = new ResponseManager(this.planManager) } } class ResponseManager { - generate = (responsePayload) => { - return "Response payloads pending..." + constructor(planManager) { + this.planManager = planManager + } + generate = (searchFilters, plan, results) => { + // generates the desc for an answered question + // start with the statement from the dropdown + let desc = this.planManager.expressPlan(plan) + if (desc === "") return desc + // add filter info + if (!Object.keys(searchFilters).length) { + desc += " across all available data" + } else { + desc += " for data entries in which " + + // this is going to be a bit of a hack for now as searchFilters in results + // apparently don't include the entity they're housed on + const attrExpressions = Object.assign({}, ...Object.entries(this.planManager.nicenameMap.fields).map(entry => entry[1])) + + Object.entries(searchFilters).forEach((filt, idx) => { + if (idx != 0) desc += " and " + desc += `${attrExpressions[filt[0]][0]} contains "${filt[1]}"` + }) + } + + // also, append the results if there won't be any further info... + if (results.results.length === 0) { + desc += ` couldn't be generated.` + } else if (results.results.length === 1) { + const fresult = (isNaN(Number(results.results[0]))) ? results.units.results[0] : Number(results.results[0]).toLocaleString() + desc += ` is ${fresult}` + if (results.units?.results && !["count", "averageCount"].includes(plan.op)) desc += ` ${results.units.results[0]}` + desc += "." + } else { + desc += ":" + } + + return desc + } + format = (plan, results) => { + // return null if there is no data to render because the generate() statement is covering it... + if (results.results.length < 2) return null + // okay, first check to see if the results have ids + references that need to be merged... + const merged = this.mergeIdsAndReferences(results.results, results.fieldNames) + const cleanResults = merged[0] + const cleanFields = merged[1] + + let formattedResults = [] + if (cleanResults[0].length === 2) { + // simple list of tuples, assume [label, value] + formattedResults = cleanResults.map(entry => {return {label: entry[0], value: entry[1]}}) + } else if (cleanResults[0].length === 3) { + // list of 3 = assume [series/group, label, value] + let resultsMap = {} + cleanResults.forEach(entry => { + if (!(entry[0] in resultsMap)) resultsMap[entry[0]] = [] + resultsMap[entry[0]].push({label: entry[1], value: entry[2]}) + }) + formattedResults = Object.entries(resultsMap).map(entry => {return {series: entry[0], data: entry[1]}}) + } else { + // (results.results[0].length > 3) only happens with multi group bys + // come back and TODO + debugger + return null + } + + return { + data: formattedResults, + visType: this.pickVisType(plan, formattedResults) + } + } + mergeIdsAndReferences = (results, fields) => { + // checks results[items] of 3 or more to see if two of the keys are really and id and reference for the same thing + // if found, merge them... + // results here is a list of tuples and fields is a list of objects of metadata + if (fields.length < 3) return [results, fields] + // do we have to merge? + // how many ids? + const idIndexes = fields.map(field => field.field).map((e, i) => e === "id" ? i : '').filter(String) + if (idIndexes.length === 0) return [results, fields] + // if ids, how many references? + const refIndexes = fields.map(field => field.field).map((e, i) => e === "reference" ? i : '').filter(String) + if (refIndexes.length === 0) return [results, fields] + + // for every id index, check if the next thing is a refIndex + const matchedIds = idIndexes.filter((indx, i) => (refIndexes[i] === indx+1)) + if (matchedIds.length === 0) return [results, fields] + + // okay, so we have some merging to do in both all results entries + in fields + const newResults = results.map(result => { + matchedIds.forEach(idx => { + const merger = result.slice(idx, idx+2) + const replacer = `${merger[1]} (${merger[0]})` + result[idx] = replacer + result[idx+1] = "_PLACEHOLDER_" + }) + return result.filter(step => step !== "_PLACEHOLDER_") + }) + + matchedIds.forEach(idx => { + const merger = fields.slice(idx, idx+2) + merger[1].field = "reference+id" + fields[idx] = merger[1] + fields[idx+1] = "_PLACEHOLDER_" + }) + + fields = fields.filter(step => step !== "_PLACEHOLDER_") + + return [newResults, fields] + } + + pickVisType = (plan, formattedResults) => { + // options currently are bar, line and multiline...maybe groupedBar, stackedBar, scatter, geoMap and more later? + // line or multiline if timeseries is present... + if (plan.timeSeries) { + // is there a group and data key in the entries? it's multiline + if (formattedResults[0].series) return "multiline" + return "line" + } + if (formattedResults[0].series) return "groupedBar" + return "bar" } } @@ -222,14 +303,41 @@ class PlanManager { const requirements = this.operations[op].required if (!(requirements?.target?.validInputs || []).includes(attr.type)) return; - // next two are temp hacks -- skip ops that require more than a single target and/or parameters - // have to deal with percentage/correlations, average counts, etc - if ((requirements?.target?.parameters || []).length > 0) return; - if (Object.keys(requirements).length > 1 || !Object.keys(requirements).includes("target")) return; + // are there "per" fields to add? + if (Object.keys(requirements).includes("per")) { + // per fields always use relationships + if (!relationship) { + // means this is the target entity + // iterate over the rest of the self.analysisSpace keys besides self + // if relationship type makes sense (e.g. "m2m" or "m2o"), then add a plan + Object.entries(this.analysisSpace) + .filter(entry => !(entry[0] === "_self")) + .filter(entry => ["m2o","m2m"].includes(entry[1].relType)) + .forEach(entry => { + planSet.push(this._generateSpecialPlan("per", basePlanTemplate, op, entry)) + }) + } else { + // means this is a related entity to the target + // only tie this back to _self (the target) + // but only if relationship type makes sense the other way (e.g. "m2m" or "o2m") + Object.entries(this.analysisSpace) + .filter(entry => (entry[0] === "_self")) + .filter(entry => ["o2m","m2m"].includes(entry[1].relType)) + .forEach(entry => { + planSet.push(this._generateSpecialPlan("per", basePlanTemplate, op, entry)) + }) + } + } else { + // just a one-off plan...finish/add it + + // next one is a temp hack -- skip ops that require more than a single target and/or parameters + // have to deal with percentage/correlations, average counts, etc + if (["percentage", "distribution", "oneHot"].includes(op)) return; - let newPlan = JSON.parse(JSON.stringify(basePlanTemplate)) - newPlan.op = op - planSet.push(newPlan) + let newPlan = JSON.parse(JSON.stringify(basePlanTemplate)) + newPlan.op = op + planSet.push(newPlan) + } }) // manage ops with additional requirements @@ -333,13 +441,31 @@ class PlanManager { return updatedPlan }).flat() + _generateSpecialPlan = (planType, basePlanTemplate, op, branch) => { + let newPlan = JSON.parse(JSON.stringify(basePlanTemplate)) + newPlan.op = op + if (planType == "per") { + newPlan.per = { + "field": "id", + "entity": branch[1].entity + } + newPlan.relationships.push(branch[0]) + } else { + // others? + } + return newPlan + } + expressPlan = (plan) => { let opTemplate = this.nicenameMap.operations[plan.op] - const pluralPicker = (["count"].includes(plan.op)) ? 1 : 0; + const pluralPicker = (["count", "averageCount", "averageSum"].includes(plan.op)) ? 1 : 0; + if (!opTemplate) return null; [...opTemplate.matchAll(this.templateTokenMatcher)].forEach(match => { if (match[1] == "target") { opTemplate = opTemplate.replace(match[0], this.nicenameMap.fields[plan.target.entity][plan.target.field][pluralPicker]) + } else if (match[1] == "per") { + opTemplate = opTemplate.replace(match[0], this.nicenameMap.fields[plan.per.entity][plan.per.field][0]) } else { console.log(match[1]) // TODO: implementation for other types of slot-fillers (e.g. plans with "per" slots) @@ -352,19 +478,31 @@ class PlanManager { if (plan.timeSeries) { const ts = plan.timeSeries if (plan.target.entity === ts.entity) { - opTemplate = `${opTemplate} over time by ${this.nicenameMap.fields[ts.entity][ts.field][0]}` + opTemplate = `${opTemplate} over time (by ${this.nicenameMap.fields[ts.entity][ts.field][0]})` } else { - opTemplate = `${opTemplate} over time by ${ts.entity}'s ${this.nicenameMap.fields[ts.entity][ts.field][0]}` + opTemplate = `${opTemplate} over time (by ${ts.entity}'s ${this.nicenameMap.fields[ts.entity][ts.field][0]})` } } if (plan.groupBy) { // might be multiple group bys so chain them... - const groupBys = plan.groupBy.map(gb => - (gb.entity === plan.target.entity) ? - `grouped by ${this.nicenameMap.fields[gb.entity][gb.field][0]}` - : `grouped by ${this.nicenameMap.fields[gb.entity][gb.field]} of ${gb.entity}` - ).join(" ") + const groupBys = plan.groupBy.map(gb => { + let gbStatement = `grouped by ${this.nicenameMap.fields[gb.entity][gb.field][0]}` + if (gb.entity !== plan.target.entity) { + // maybe mention the fact the groupBy is across a different entity? + let entityMentionTest = true + const entityName = this.nicenameMap.fields[gb.entity].id[0] + // unless... + // a) that's already been stated (because it's already "group by ") + if (gb.field === "id") entityMentionTest = false + // b) the "per" field involves the second entity (?) + if (gb.entity === plan.per?.entity) entityMentionTest = false + // c) the nicename of the attribute already mentions the entity + if (gbStatement.toLowerCase().search(entityName.toLowerCase())) entityMentionTest = false + + if (entityMentionTest) gbStatement += ` (of ${entityName})` + } + }).join(" ") opTemplate = `${opTemplate} ${groupBys}` }