Merge pull request #828 from openedx/cag/query-context

feat: serialize query_context as json for redability
openedx · May 27, 2024 · a81de2a · a81de2a
2 parents d99218b + 714e660
commit a81de2a
Show file tree

Hide file tree

Showing 63 changed files with 5,870 additions and 574 deletions.
diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py
@@ -5,6 +5,7 @@
 import glob
 import os
 import re
+import json
 from zipfile import ZipFile
 
 import click
@@ -171,6 +172,9 @@ class ChartAsset(Asset):
     def process(self, content: dict, existing: dict):
         if not content.get("query_context"):
             content["query_context"] = existing.get("query_context")
+        query_context = content["query_context"]
+        if query_context is not None and isinstance(query_context, str):
+            content["query_context"] = json.loads(query_context)
 
 
 class DashboardAsset(Asset):

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets_utils.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets_utils.py
@@ -24,6 +24,7 @@
 # version, which skips a variety of other checks.
 import logging
 import yaml
+import json
 from pathlib import Path
 
 from flask import g
@@ -57,7 +58,13 @@ def load_configs_from_directory(
             queue.extend(path_name.glob("*"))
         elif path_name.suffix.lower() in YAML_EXTENSIONS:
             with open(path_name) as fp:
-                contents[str(path_name.relative_to(root))] = fp.read()
+                content = fp.read()
+                loaded_content = yaml.load(content, Loader=yaml.Loader)
+                if loaded_content.get("query_context") and isinstance(loaded_content["query_context"], dict):
+                    loaded_content["query_context"] = json.dumps(loaded_content["query_context"])
+                content = yaml.dump(loaded_content)
+                contents[str(path_name.relative_to(root))] = content
+
 
     # removing "type" from the metadata allows us to import any exported model
     # from the unzipped directory directly

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py
@@ -140,7 +140,7 @@ def get_query_contexts_from_assets():
             with open(path, "r") as file:
                 asset = yaml.safe_load(file)
                 if "query_context" in asset and asset["query_context"]:
-                    query_contexts[asset["uuid"]] = json.loads(asset["query_context"])
+                    query_contexts[asset["uuid"]] = asset["query_context"]
 
     logger.info(f"Found {len(query_contexts)} query contexts")
     return query_contexts

diff --git a/...templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Courses.yaml b/...templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Courses.yaml
@@ -59,13 +59,130 @@ params:
   y_axis_title_margin: 15
   y_axis_title_position: Left
   zoomable: true
-query_context: '{"datasource":{"id":829,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_hour","op":"TEMPORAL_RANGE","val":"No
-  filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_hour","label":"emission_hour","expressionType":"SQL"}],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":true,"label":"Active
-  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":"uniqCombinedMerge(courses_cnt)"}],"orderby":[[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":true,"label":"Active
-  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":"uniqCombinedMerge(courses_cnt)"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_hour"],"columns":[],"aggregates":{"Active
-  Courses":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"829__table","viz_type":"echarts_timeseries_line","slice_id":3402,"x_axis":"emission_hour","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":true,"label":"Active
-  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":"uniqCombinedMerge(courses_cnt)"}],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
-  filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_hour"}],"order_desc":true,"row_limit":10000,"truncate_metric":false,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","only_total":true,"opacity":0.2,"markerEnabled":false,"markerSize":6,"zoomable":true,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","truncateXAxis":false,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[2422],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+query_context:
+  datasource:
+    id: 829
+    type: table
+  force: false
+  form_data:
+    adhoc_filters:
+    - clause: WHERE
+      comparator: No filter
+      expressionType: SIMPLE
+      operator: TEMPORAL_RANGE
+      subject: emission_hour
+    annotation_layers: []
+    color_scheme: supersetColors
+    comparison_type: values
+    dashboards:
+    - 2422
+    datasource: 829__table
+    extra_form_data: {}
+    force: false
+    forecastInterval: 0.8
+    forecastPeriods: 10
+    groupby: []
+    legendMargin: null
+    legendOrientation: top
+    legendType: scroll
+    markerEnabled: false
+    markerSize: 6
+    metrics:
+    - aggregate: null
+      column: null
+      datasourceWarning: false
+      expressionType: SQL
+      hasCustomLabel: true
+      label: Active Courses
+      optionName: metric_hvxlgfij23e_68a6guxh855
+      sqlExpression: uniqCombinedMerge(courses_cnt)
+    only_total: true
+    opacity: 0.2
+    order_desc: true
+    result_format: json
+    result_type: full
+    rich_tooltip: true
+    row_limit: 10000
+    seriesType: line
+    show_empty_columns: true
+    show_legend: false
+    slice_id: 3402
+    sort_series_type: sum
+    time_grain_sqla: P1D
+    tooltipTimeFormat: smart_date
+    truncateXAxis: false
+    truncate_metric: false
+    viz_type: echarts_timeseries_line
+    x_axis: emission_hour
+    x_axis_sort_asc: true
+    x_axis_sort_series: name
+    x_axis_sort_series_ascending: true
+    x_axis_time_format: '%Y-%m-%d'
+    x_axis_title_margin: 15
+    y_axis_bounds:
+    - null
+    - null
+    y_axis_format: SMART_NUMBER
+    y_axis_title_margin: 15
+    y_axis_title_position: Left
+    zoomable: true
+  queries:
+  - annotation_layers: []
+    applied_time_extras: {}
+    columns:
+    - columnType: BASE_AXIS
+      expressionType: SQL
+      label: emission_hour
+      sqlExpression: emission_hour
+      timeGrain: P1D
+    custom_form_data: {}
+    custom_params: {}
+    extras:
+      having: ''
+      time_grain_sqla: P1D
+      where: ''
+    filters:
+    - col: emission_hour
+      op: TEMPORAL_RANGE
+      val: No filter
+    metrics:
+    - aggregate: null
+      column: null
+      datasourceWarning: false
+      expressionType: SQL
+      hasCustomLabel: true
+      label: Active Courses
+      optionName: metric_hvxlgfij23e_68a6guxh855
+      sqlExpression: uniqCombinedMerge(courses_cnt)
+    order_desc: true
+    orderby:
+    - - aggregate: null
+        column: null
+        datasourceWarning: false
+        expressionType: SQL
+        hasCustomLabel: true
+        label: Active Courses
+        optionName: metric_hvxlgfij23e_68a6guxh855
+        sqlExpression: uniqCombinedMerge(courses_cnt)
+      - false
+    post_processing:
+    - operation: pivot
+      options:
+        aggregates:
+          Active Courses:
+            operator: mean
+        columns: []
+        drop_missing_columns: false
+        index:
+        - emission_hour
+    - operation: flatten
+    row_limit: 10000
+    series_columns: []
+    series_limit: 0
+    time_offsets: []
+    url_params: {}
+  result_format: json
+  result_type: full
 slice_name: Active Courses
 uuid: 138eb72a-f73f-42b5-96d6-3417715c3519
 version: 1.0.0

diff --git a/...emplates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Learners.yaml b/...emplates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Learners.yaml
@@ -63,13 +63,134 @@ params:
   y_axis_title_margin: 15
   y_axis_title_position: Left
   zoomable: true
-query_context: '{"datasource":{"id":84,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_hour","op":"TEMPORAL_RANGE","val":"No
-  filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_hour","label":"emission_hour","expressionType":"SQL"}],"metrics":[{"expressionType":"SQL","sqlExpression":"uniqCombinedMerge(actors_cnt)","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Active
-  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"uniqCombinedMerge(actors_cnt)","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Active
-  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_hour"],"columns":[],"aggregates":{"Active
-  Learners":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"84__table","viz_type":"echarts_timeseries_line","slice_id":3403,"x_axis":"emission_hour","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"uniqCombinedMerge(actors_cnt)","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Active
-  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855"}],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
-  filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_hour"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","only_total":true,"area":false,"opacity":0.2,"markerEnabled":false,"markerSize":6,"zoomable":true,"minorTicks":false,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","minorSplitLine":false,"truncateXAxis":false,"truncateYAxis":false,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[2422],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+query_context:
+  datasource:
+    id: 84
+    type: table
+  force: false
+  form_data:
+    adhoc_filters:
+    - clause: WHERE
+      comparator: No filter
+      expressionType: SIMPLE
+      operator: TEMPORAL_RANGE
+      subject: emission_hour
+    annotation_layers: []
+    area: false
+    color_scheme: supersetColors
+    comparison_type: values
+    dashboards:
+    - 2422
+    datasource: 84__table
+    extra_form_data: {}
+    force: false
+    forecastInterval: 0.8
+    forecastPeriods: 10
+    groupby: []
+    legendMargin: null
+    legendOrientation: top
+    legendType: scroll
+    markerEnabled: false
+    markerSize: 6
+    metrics:
+    - aggregate: null
+      column: null
+      datasourceWarning: false
+      expressionType: SQL
+      hasCustomLabel: true
+      label: Active Learners
+      optionName: metric_hvxlgfij23e_68a6guxh855
+      sqlExpression: uniqCombinedMerge(actors_cnt)
+    minorSplitLine: false
+    minorTicks: false
+    only_total: true
+    opacity: 0.2
+    order_desc: true
+    result_format: json
+    result_type: full
+    rich_tooltip: true
+    row_limit: 10000
+    seriesType: line
+    show_empty_columns: true
+    show_legend: false
+    slice_id: 3403
+    sort_series_type: sum
+    time_grain_sqla: P1D
+    tooltipTimeFormat: smart_date
+    truncateXAxis: false
+    truncateYAxis: false
+    truncate_metric: true
+    viz_type: echarts_timeseries_line
+    x_axis: emission_hour
+    x_axis_sort_asc: true
+    x_axis_sort_series: name
+    x_axis_sort_series_ascending: true
+    x_axis_time_format: '%Y-%m-%d'
+    x_axis_title_margin: 15
+    y_axis_bounds:
+    - null
+    - null
+    y_axis_format: SMART_NUMBER
+    y_axis_title_margin: 15
+    y_axis_title_position: Left
+    zoomable: true
+  queries:
+  - annotation_layers: []
+    applied_time_extras: {}
+    columns:
+    - columnType: BASE_AXIS
+      expressionType: SQL
+      label: emission_hour
+      sqlExpression: emission_hour
+      timeGrain: P1D
+    custom_form_data: {}
+    custom_params: {}
+    extras:
+      having: ''
+      time_grain_sqla: P1D
+      where: ''
+    filters:
+    - col: emission_hour
+      op: TEMPORAL_RANGE
+      val: No filter
+    metrics:
+    - aggregate: null
+      column: null
+      datasourceWarning: false
+      expressionType: SQL
+      hasCustomLabel: true
+      label: Active Learners
+      optionName: metric_hvxlgfij23e_68a6guxh855
+      sqlExpression: uniqCombinedMerge(actors_cnt)
+    order_desc: true
+    orderby:
+    - - aggregate: null
+        column: null
+        datasourceWarning: false
+        expressionType: SQL
+        hasCustomLabel: true
+        label: Active Learners
+        optionName: metric_hvxlgfij23e_68a6guxh855
+        sqlExpression: uniqCombinedMerge(actors_cnt)
+      - false
+    post_processing:
+    - operation: pivot
+      options:
+        aggregates:
+          Active Learners:
+            operator: mean
+        columns: []
+        drop_missing_columns: false
+        index:
+        - emission_hour
+    - operation: flatten
+    row_limit: 10000
+    series_columns: []
+    series_limit: 0
+    time_offsets: []
+    url_params: {}
+  result_format: json
+  result_type: full
 slice_name: Active Learners
 uuid: b9b602ec-001b-4584-aa24-9201f3b8746f
 version: 1.0.0