From 6c9f957290a2ffde73a8acacd496d1fe185f4451 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 3 Oct 2023 15:04:37 -0400 Subject: [PATCH 1/3] refactor: Tidy up Operator Dashboard - Make names consistent - Remove unusable or useless charts - Add a "slowest ClickHouse queries" chart --- .gitignore | 1 + ...yaml => Course_Enrollments_Over_Time.yaml} | 25 +- .../charts/Slowest_ClickHouse_Queries.yaml | 43 ++ .../assets/dashboards/Operator_Dashboard.yaml | 398 +++++++++++------- .../datasets/slowest_clickhouse_queries.yaml | 91 ++++ 5 files changed, 395 insertions(+), 163 deletions(-) rename tutoraspects/templates/openedx-assets/assets/charts/{Course_Registrations_Over_Time.yaml => Course_Enrollments_Over_Time.yaml} (78%) create mode 100644 tutoraspects/templates/openedx-assets/assets/charts/Slowest_ClickHouse_Queries.yaml create mode 100644 tutoraspects/templates/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml diff --git a/.gitignore b/.gitignore index 5902e96d2..50cd6869f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ !.gitignore TODO __pycache__ +*.DS_Store* *.egg-info/ /build/ /dist/ diff --git a/tutoraspects/templates/openedx-assets/assets/charts/Course_Registrations_Over_Time.yaml b/tutoraspects/templates/openedx-assets/assets/charts/Course_Enrollments_Over_Time.yaml similarity index 78% rename from tutoraspects/templates/openedx-assets/assets/charts/Course_Registrations_Over_Time.yaml rename to tutoraspects/templates/openedx-assets/assets/charts/Course_Enrollments_Over_Time.yaml index 5f651dab1..46c75ba74 100644 --- a/tutoraspects/templates/openedx-assets/assets/charts/Course_Registrations_Over_Time.yaml +++ b/tutoraspects/templates/openedx-assets/assets/charts/Course_Enrollments_Over_Time.yaml @@ -1,4 +1,4 @@ -_file_name: Course_Registrations_Over_Time.yaml +_file_name: Course_Enrollments_Over_Time.yaml cache_timeout: null certification_details: null certified_by: null @@ -16,14 +16,20 @@ params: operatorId: EQUALS sqlExpression: null subject: enrollment_status + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: enrollment_status_date annotation_layers: [] color_scheme: supersetColors comparison_type: values - datasource: 2__table + dashboards: + - 14 + datasource: 1__table extra_form_data: {} forecastInterval: 0.8 forecastPeriods: 10 - granularity_sqla: enrollment_status_date groupby: - enrollment_status legendOrientation: top @@ -55,16 +61,23 @@ params: optionName: metric_iapetcrge8_4fkcsr2b3nj sqlExpression: null only_total: true + opacity: 0.2 order_desc: true rich_tooltip: true row_limit: 10000 + seriesType: line + show_empty_columns: true show_legend: true - slice_id: 451 + slice_id: 272 + sort_series_type: sum time_grain_sqla: P1D - time_range: No filter tooltipTimeFormat: smart_date truncate_metric: true viz_type: echarts_timeseries_line + x_axis: enrollment_status_date + x_axis_sort_asc: true + x_axis_sort_series: name + x_axis_sort_series_ascending: true x_axis_time_format: smart_date x_axis_title_margin: 15 y_axis_bounds: @@ -75,7 +88,7 @@ params: y_axis_title_margin: 15 y_axis_title_position: Left query_context: null -slice_name: Course Registrations Over Time +slice_name: Course Enrollments Over Time uuid: bf4f4671-c276-4185-9b9a-b10864efea6c version: 1.0.0 viz_type: echarts_timeseries_line diff --git a/tutoraspects/templates/openedx-assets/assets/charts/Slowest_ClickHouse_Queries.yaml b/tutoraspects/templates/openedx-assets/assets/charts/Slowest_ClickHouse_Queries.yaml new file mode 100644 index 000000000..63d9bbfe7 --- /dev/null +++ b/tutoraspects/templates/openedx-assets/assets/charts/Slowest_ClickHouse_Queries.yaml @@ -0,0 +1,43 @@ +_file_name: Slowest_ClickHouse_Queries.yaml +cache_timeout: null +certification_details: null +certified_by: null +dataset_uuid: e8748c23-6220-413d-bd61-de8306163e10 +description: null +params: + adhoc_filters: + - clause: WHERE + comparator: No filter + expressionType: SIMPLE + operator: TEMPORAL_RANGE + subject: event_time + all_columns: + - event_time + - duration_secs + - read_rows + - memory_usage_kb + - query + color_pn: true + dashboards: [] + datasource: 21__table + extra_form_data: {} + groupby: [] + metrics: [] + order_by_cols: + - '["duration_secs", false]' + order_desc: true + percent_metrics: [] + query_mode: raw + row_limit: 100 + server_page_length: 10 + show_cell_bars: true + table_timestamp_format: smart_date + temporal_columns_lookup: + event_time: true + time_grain_sqla: P1D + viz_type: table +query_context: null +slice_name: Slowest ClickHouse Queries +uuid: 953af3e5-8fc6-441e-bff1-22987735edf0 +version: 1.0.0 +viz_type: table diff --git a/tutoraspects/templates/openedx-assets/assets/dashboards/Operator_Dashboard.yaml b/tutoraspects/templates/openedx-assets/assets/dashboards/Operator_Dashboard.yaml index 45d4969b2..d068a62b0 100644 --- a/tutoraspects/templates/openedx-assets/assets/dashboards/Operator_Dashboard.yaml +++ b/tutoraspects/templates/openedx-assets/assets/dashboards/Operator_Dashboard.yaml @@ -5,18 +5,159 @@ css: '' dashboard_title: Operator Dashboard description: null metadata: - chart_configuration: {} - color_scheme: '' - color_scheme_domain: [] + chart_configuration: + '240': + crossFilters: + chartsInScope: + - 74 + - 176 + - 186 + - 206 + - 253 + - 260 + - 269 + - 272 + - 275 + - 285 + - 297 + - 383 + scope: global + id: 240 + '253': + crossFilters: + chartsInScope: + - 74 + - 176 + - 186 + - 206 + - 240 + - 260 + - 269 + - 272 + - 275 + - 285 + - 297 + - 383 + scope: global + id: 253 + '272': + crossFilters: + chartsInScope: + - 74 + - 176 + - 186 + - 206 + - 240 + - 253 + - 260 + - 269 + - 275 + - 285 + - 297 + - 383 + scope: global + id: 272 + '275': + crossFilters: + chartsInScope: + - 74 + - 176 + - 186 + - 206 + - 240 + - 253 + - 260 + - 269 + - 272 + - 285 + - 297 + - 383 + scope: global + id: 275 + '383': + crossFilters: + chartsInScope: + - 74 + - 176 + - 186 + - 206 + - 240 + - 253 + - 260 + - 269 + - 272 + - 275 + - 285 + - 297 + scope: global + id: 383 + '74': + crossFilters: + chartsInScope: + - 176 + - 186 + - 206 + - 240 + - 253 + - 260 + - 269 + - 272 + - 275 + - 285 + - 297 + - 383 + scope: global + id: 74 + color_scheme: supersetColors + color_scheme_domain: + - '#1FA8C9' + - '#454E7C' + - '#5AC189' + - '#FF7F44' + - '#666666' + - '#E04355' + - '#FCC700' + - '#A868B7' + - '#3CCCCB' + - '#A38F79' + - '#8FD3E4' + - '#A1A6BD' + - '#ACE1C4' + - '#FEC0A1' + - '#B2B2B2' + - '#EFA1AA' + - '#FDE380' + - '#D3B3DA' + - '#9EE5E5' + - '#D1C6BC' cross_filters_enabled: false default_filters: '{}' expanded_slices: {} + global_chart_configuration: + chartsInScope: + - 74 + - 176 + - 186 + - 206 + - 240 + - 253 + - 260 + - 269 + - 272 + - 275 + - 285 + - 297 + - 383 + scope: + excluded: [] + rootPath: + - ROOT_ID label_colors: {} native_filter_configuration: - cascadeParentIds: [] chartsInScope: - - 451 - - 452 + - 240 + - 272 controlValues: defaultToFirstItem: false enableEmptyFilter: false @@ -44,8 +185,8 @@ metadata: type: NATIVE_FILTER - cascadeParentIds: [] chartsInScope: - - 453 - - 454 + - 240 + - 272 controlValues: defaultToFirstItem: false enableEmptyFilter: false @@ -65,7 +206,7 @@ metadata: rootPath: - TAB-uKjvZ62aJt tabsInScope: - - TAB-6Mdnw3FZh + - TAB-uKjvZ62aJt targets: - column: name: course_run @@ -73,21 +214,22 @@ metadata: type: NATIVE_FILTER - cascadeParentIds: [] chartsInScope: - - 411 - - 413 - - 427 - - 430 - - 437 - - 438 - - 445 - - 450 - - 454 + - 74 + - 176 + - 186 + - 206 + - 260 + - 269 + - 275 + - 285 + - 383 controlValues: defaultToFirstItem: false enableEmptyFilter: false inverseSelection: false multiSelect: false searchAllOptions: false + sortAscending: true defaultDataMask: extraFormData: {} filterState: {} @@ -98,37 +240,35 @@ metadata: name: Course Key scope: excluded: - - 116 - - 124 - - 180 - - 13 + - 297 rootPath: - - ROOT_ID + - TAB-DE73B5pXm + - TAB-6Mdnw3FZh tabsInScope: - TAB-6Mdnw3FZh - TAB-DE73B5pXm + - TAB-EygpebMa1 + - TAB-gvqU89mvT targets: - column: name: course_key - datasetUuid: 2a2498dc-03ce-41a0-b798-d84f808f7da6 + datasetUuid: 41278a97-d0ff-4645-9514-d79f80d275df type: NATIVE_FILTER refresh_frequency: 0 shared_label_colors: - attempted: '#5AC189' - earned: '#1FA8C9' - evaluated: '#454E7C' - registered: '#FF7F44' + course-v1:edX+DemoX+Demo_Course: '#1FA8C9' + edX: '#454E7C' timed_refresh_immune_slices: [] position: CHART-8rDUkRBgv9: children: [] id: CHART-8rDUkRBgv9 meta: - chartId: 180 + chartId: 240 height: 50 sliceName: Enrollments By Type uuid: 6c473b09-6a1a-4531-b71d-ab82e09721ef - width: 4 + width: 6 parents: - ROOT_ID - GRID_ID @@ -140,11 +280,11 @@ position: children: [] id: CHART-H4NTUWfMqH meta: - chartId: 124 + chartId: 272 height: 50 - sliceName: Course Registrations Over Time + sliceName: Course Enrollments Over Time uuid: bf4f4671-c276-4185-9b9a-b10864efea6c - width: 4 + width: 6 parents: - ROOT_ID - GRID_ID @@ -152,11 +292,27 @@ position: - TAB-uKjvZ62aJt - ROW-VQVoPfsLlS type: CHART + CHART-La3O2IbbVV: + children: [] + id: CHART-La3O2IbbVV + meta: + chartId: 383 + height: 50 + sliceName: Slowest ClickHouse Queries + uuid: 953af3e5-8fc6-441e-bff1-22987735edf0 + width: 12 + parents: + - ROOT_ID + - GRID_ID + - TABS-7nA6MwltSD + - TAB-gvqU89mvT + - ROW-cpoIgMfQUZ + type: CHART CHART-QN5x3ibIzS: children: [] id: CHART-QN5x3ibIzS meta: - chartId: 313 + chartId: 74 height: 50 sliceName: Active Users Per Organization uuid: 09a8a31c-c65f-452a-951f-dce6f0f03292 @@ -172,57 +328,56 @@ position: children: [] id: CHART-explore-15-1 meta: - chartId: 284 + chartId: 176 height: 50 sliceName: xAPI Events Over Time uuid: e6a30923-6382-413e-a1c7-2ac223fa5c5c - width: 4 + width: 6 parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - TAB-DE73B5pXm - - ROW-7gxdpYV-y + - ROW-X7WD5wVoP type: CHART CHART-explore-17-1: children: [] id: CHART-explore-17-1 meta: - chartId: 12 + chartId: 285 height: 16 sliceName: Last Received Event uuid: ffdaabc5-8d01-47d7-8f29-66efa643befb - width: 2 + width: 3 parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - TAB-DE73B5pXm - ROW-mPSz65tZ7E - - COLUMN-eGzPLamIzw type: CHART CHART-explore-19-1: children: [] id: CHART-explore-19-1 meta: - chartId: 116 + chartId: 297 height: 16 sliceName: Last course syncronized + sliceNameOverride: Last Course Published uuid: ca93f427-d021-476e-9a36-6ea0d99d30ea - width: 1 + width: 3 parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - TAB-DE73B5pXm - ROW-mPSz65tZ7E - - COLUMN-eGzPLamIzw type: CHART CHART-explore-20-1: children: [] id: CHART-explore-20-1 meta: - chartId: 176 + chartId: 260 height: 14 sliceName: Total Organizations uuid: e66b1cdb-6974-4fa2-a769-4908665118f6 @@ -233,15 +388,16 @@ position: - TABS-7nA6MwltSD - TAB-DE73B5pXm - ROW-mPSz65tZ7E - - COLUMN-GDahqg06nQ + - COLUMN-eGzPLamIzw type: CHART CHART-explore-21-1: children: [] id: CHART-explore-21-1 meta: - chartId: 316 + chartId: 186 height: 16 sliceName: Unique actors + sliceNameOverride: Total Unique Users uuid: 071bf3d5-8b9d-4e26-83fb-7836db5a7ab6 width: 2 parents: @@ -250,15 +406,16 @@ position: - TABS-7nA6MwltSD - TAB-DE73B5pXm - ROW-mPSz65tZ7E - - COLUMN-GDahqg06nQ + - COLUMN-eGzPLamIzw type: CHART CHART-explore-22-1: children: [] id: CHART-explore-22-1 meta: - chartId: 264 + chartId: 269 height: 15 sliceName: Courses + sliceNameOverride: Total Courses uuid: 63f86926-3325-43cf-b075-ef37f5f60413 width: 2 parents: @@ -267,66 +424,34 @@ position: - TABS-7nA6MwltSD - TAB-DE73B5pXm - ROW-mPSz65tZ7E - - COLUMN-GDahqg06nQ - type: CHART - CHART-explore-25-1: - children: [] - id: CHART-explore-25-1 - meta: - chartId: 162 - height: 50 - sliceName: Event type - uuid: b4a9a10d-0e82-43e7-a619-3ef3f6bf57a5 - width: 4 - parents: - - ROOT_ID - - GRID_ID - - TABS-7nA6MwltSD - - TAB-DE73B5pXm - - ROW-mPSz65tZ7E - type: CHART - CHART-explore-26-1: - children: [] - id: CHART-explore-26-1 - meta: - chartId: 41 - height: 50 - sliceName: Events per course - uuid: a445d9e5-ac98-411e-9a47-eda10081bd5b - width: 4 - parents: - - ROOT_ID - - GRID_ID - - TABS-7nA6MwltSD - - TAB-DE73B5pXm - - ROW-7gxdpYV-y + - COLUMN-eGzPLamIzw type: CHART CHART-explore-27-1: children: [] id: CHART-explore-27-1 meta: - chartId: 262 + chartId: 206 height: 50 sliceName: Actor IDs over time sliceNameOverride: Active Users Over Time uuid: 6a8ded91-f9a6-45c2-9f9f-f85dc1ff7ed6 - width: 4 + width: 6 parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - TAB-DE73B5pXm - - ROW-7gxdpYV-y + - ROW-X7WD5wVoP type: CHART CHART-o87Pu6wg5_: children: [] id: CHART-o87Pu6wg5_ meta: - chartId: 22 + chartId: 275 height: 50 sliceName: Most Active Courses Per Day uuid: a19a7c6a-e2c9-4033-86b3-b2e5878b5b69 - width: 4 + width: 12 parents: - ROOT_ID - GRID_ID @@ -338,7 +463,7 @@ position: children: [] id: CHART-wz1Y-JkV9- meta: - chartId: 13 + chartId: 253 height: 50 sliceName: Courses Per Organization uuid: 7901d6ae-dea8-4d2c-9232-35c22571cfb7 @@ -352,9 +477,7 @@ position: type: CHART COLUMN-GDahqg06nQ: children: - - CHART-explore-21-1 - - CHART-explore-22-1 - - CHART-explore-20-1 + - MARKDOWN-EZXXAbo-Tg id: COLUMN-GDahqg06nQ meta: background: BACKGROUND_TRANSPARENT @@ -368,12 +491,13 @@ position: type: COLUMN COLUMN-eGzPLamIzw: children: - - CHART-explore-17-1 - - CHART-explore-19-1 + - CHART-explore-21-1 + - CHART-explore-22-1 + - CHART-explore-20-1 id: COLUMN-eGzPLamIzw meta: background: BACKGROUND_TRANSPARENT - width: 4 + width: 2 parents: - ROOT_ID - GRID_ID @@ -398,32 +522,6 @@ position: parents: - ROOT_ID type: GRID - HEADER-GcKR2Le6_Z: - children: [] - id: HEADER-GcKR2Le6_Z - meta: - background: BACKGROUND_TRANSPARENT - headerSize: MEDIUM_HEADER - text: Coming Soon - parents: - - ROOT_ID - - GRID_ID - - TABS-7nA6MwltSD - - TAB-pkVFpoQC- - type: HEADER - HEADER-lsKtwx26sE: - children: [] - id: HEADER-lsKtwx26sE - meta: - background: BACKGROUND_TRANSPARENT - headerSize: MEDIUM_HEADER - text: Coming Soon - parents: - - ROOT_ID - - GRID_ID - - TABS-7nA6MwltSD - - TAB-0QACRjilP - type: HEADER HEADER_ID: id: HEADER_ID meta: @@ -458,14 +556,15 @@ position: {% endif %} {% endfor %} " - height: 50 + height: 53 width: 4 parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - TAB-DE73B5pXm - - ROW-fgCiGD7i8F + - ROW-mPSz65tZ7E + - COLUMN-GDahqg06nQ type: MARKDOWN ROOT_ID: children: @@ -485,32 +584,31 @@ position: - TABS-7nA6MwltSD - TAB-EygpebMa1 type: ROW - ROW-7gxdpYV-y: + ROW-VQVoPfsLlS: children: - - CHART-explore-15-1 - - CHART-explore-27-1 - - CHART-explore-26-1 - id: ROW-7gxdpYV-y + - CHART-H4NTUWfMqH + - CHART-8rDUkRBgv9 + id: ROW-VQVoPfsLlS meta: background: BACKGROUND_TRANSPARENT parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - - TAB-DE73B5pXm + - TAB-uKjvZ62aJt type: ROW - ROW-VQVoPfsLlS: + ROW-X7WD5wVoP: children: - - CHART-H4NTUWfMqH - - CHART-8rDUkRBgv9 - id: ROW-VQVoPfsLlS + - CHART-explore-27-1 + - CHART-explore-15-1 + id: ROW-X7WD5wVoP meta: background: BACKGROUND_TRANSPARENT parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - - TAB-uKjvZ62aJt + - TAB-DE73B5pXm type: ROW ROW-Z0QlRyckta: children: @@ -524,23 +622,24 @@ position: - TABS-7nA6MwltSD - TAB-6Mdnw3FZh type: ROW - ROW-fgCiGD7i8F: + ROW-cpoIgMfQUZ: children: - - MARKDOWN-EZXXAbo-Tg - id: ROW-fgCiGD7i8F + - CHART-La3O2IbbVV + id: ROW-cpoIgMfQUZ meta: background: BACKGROUND_TRANSPARENT parents: - ROOT_ID - GRID_ID - TABS-7nA6MwltSD - - TAB-DE73B5pXm + - TAB-gvqU89mvT type: ROW ROW-mPSz65tZ7E: children: - - CHART-explore-25-1 - COLUMN-GDahqg06nQ - COLUMN-eGzPLamIzw + - CHART-explore-17-1 + - CHART-explore-19-1 id: ROW-mPSz65tZ7E meta: background: BACKGROUND_TRANSPARENT @@ -550,19 +649,6 @@ position: - TABS-7nA6MwltSD - TAB-DE73B5pXm type: ROW - TAB-0QACRjilP: - children: - - HEADER-lsKtwx26sE - id: TAB-0QACRjilP - meta: - defaultText: Tab title - placeholder: Tab title - text: Learner Engagement - parents: - - ROOT_ID - - GRID_ID - - TABS-7nA6MwltSD - type: TAB TAB-6Mdnw3FZh: children: - ROW-Z0QlRyckta @@ -578,9 +664,8 @@ position: type: TAB TAB-DE73B5pXm: children: - - ROW-7gxdpYV-y + - ROW-X7WD5wVoP - ROW-mPSz65tZ7E - - ROW-fgCiGD7i8F id: TAB-DE73B5pXm meta: defaultText: Tab title @@ -604,14 +689,14 @@ position: - GRID_ID - TABS-7nA6MwltSD type: TAB - TAB-pkVFpoQC-: + TAB-gvqU89mvT: children: - - HEADER-GcKR2Le6_Z - id: TAB-pkVFpoQC- + - ROW-cpoIgMfQUZ + id: TAB-gvqU89mvT meta: defaultText: Tab title placeholder: Tab title - text: Course Success + text: ClickHouse parents: - ROOT_ID - GRID_ID @@ -635,9 +720,8 @@ position: - TAB-DE73B5pXm - TAB-uKjvZ62aJt - TAB-6Mdnw3FZh - - TAB-pkVFpoQC- - - TAB-0QACRjilP - TAB-EygpebMa1 + - TAB-gvqU89mvT id: TABS-7nA6MwltSD meta: {} parents: diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml new file mode 100644 index 000000000..924120927 --- /dev/null +++ b/tutoraspects/templates/openedx-assets/assets/datasets/slowest_clickhouse_queries.yaml @@ -0,0 +1,91 @@ +_file_name: slowest_clickhouse_queries.yaml +cache_timeout: null +columns: +- advanced_data_type: null + column_name: memory_usage_kb + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Float64 + verbose_name: null +- advanced_data_type: null + column_name: duration_secs + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Float64 + verbose_name: null +- advanced_data_type: null + column_name: read_rows + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: UInt64 + verbose_name: null +- advanced_data_type: null + column_name: event_time + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: true + python_date_format: null + type: DateTime + verbose_name: null +- advanced_data_type: null + column_name: query + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: null +database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 +default_endpoint: null +description: null +extra: null +fetch_values_predicate: null +filter_select_enabled: true +main_dttm_col: event_time +metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null +normalize_columns: false +offset: 0 +params: null +schema: xapi +sql: "SELECT event_time, query_duration_ms / 1000 as duration_secs, read_rows, memory_usage\ + \ / 1024 AS memory_usage_kb, query\nFROM system.query_log\nWHERE type = 'QueryFinish'\ + \ \nORDER BY query_duration_ms DESC;\n" +table_name: slowest_clickhouse_queries +template_params: null +uuid: e8748c23-6220-413d-bd61-de8306163e10 +version: 1.0.0 From e93a0951adbafb283e7bfad05f655ba98b631554 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 10 Oct 2023 15:50:26 -0400 Subject: [PATCH 2/3] build: Move locale.yaml to new mount location Hopefully will fix k8s build issues around the size of the pythonpath configmap. --- scripts/utils.py | 4 ++-- tutoraspects/patches/k8s-deployments | 15 +++++++++++++++ tutoraspects/patches/k8s-jobs | 10 ++++++++++ .../patches/kustomization-configmapgenerator | 7 +++++++ .../{pythonpath => localization}/locale.yaml | 0 .../apps/superset/pythonpath/create_assets.py | 2 +- .../templates/base-docker-compose-services | 1 + 7 files changed, 36 insertions(+), 3 deletions(-) rename tutoraspects/templates/aspects/apps/superset/{pythonpath => localization}/locale.yaml (100%) diff --git a/scripts/utils.py b/scripts/utils.py index 20b49256e..f4f8c9216 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -27,7 +27,7 @@ def get_text_for_translations(root_path): assets_file = ( - root_path + "/env/plugins/aspects/apps/superset/pythonpath/assets.yaml" + root_path + "/env/plugins/aspects/apps/superset/localization/assets.yaml" ) strings = [] @@ -96,7 +96,7 @@ def compile_translations(root_path): tx.fetch_translations() translation_file = ( - "tutoraspects/templates/aspects/apps/superset/pythonpath/locale.yaml" + "tutoraspects/templates/aspects/apps/superset/localization/locale.yaml" ) file = open(translation_file, "w") diff --git a/tutoraspects/patches/k8s-deployments b/tutoraspects/patches/k8s-deployments index b9ddfacf0..ea750e303 100644 --- a/tutoraspects/patches/k8s-deployments +++ b/tutoraspects/patches/k8s-deployments @@ -211,6 +211,8 @@ spec: name: docker - mountPath: /app/pythonpath name: pythonpath + - mountPath: /app/localization + name: localization - mountPath: /app/security name: security volumes: @@ -220,6 +222,9 @@ spec: - name: pythonpath configMap: name: superset-pythonpath + - name: localization + configMap: + name: superset-localization - name: security configMap: name: superset-security @@ -300,6 +305,8 @@ spec: name: docker - mountPath: /app/pythonpath name: pythonpath + - mountPath: /app/localization + name: localization - mountPath: /app/security name: security volumes: @@ -309,6 +316,9 @@ spec: - name: pythonpath configMap: name: superset-pythonpath + - name: localization + configMap: + name: superset-localization - name: security configMap: name: superset-security @@ -389,6 +399,8 @@ spec: name: docker - mountPath: /app/pythonpath name: pythonpath + - mountPath: /app/localization + name: localization - mountPath: /app/security name: security volumes: @@ -398,6 +410,9 @@ spec: - name: pythonpath configMap: name: superset-pythonpath + - name: localization + configMap: + name: superset-localization - name: security configMap: name: superset-security diff --git a/tutoraspects/patches/k8s-jobs b/tutoraspects/patches/k8s-jobs index c67f45313..69013e29f 100644 --- a/tutoraspects/patches/k8s-jobs +++ b/tutoraspects/patches/k8s-jobs @@ -216,6 +216,8 @@ spec: name: docker - mountPath: /app/pythonpath name: pythonpath + - mountPath: /app/localization + name: localization - mountPath: /app/security name: security - mountPath: /app/assets @@ -227,6 +229,9 @@ spec: - name: pythonpath configMap: name: superset-pythonpath + - name: localization + configMap: + name: superset-localization - name: security configMap: name: superset-security @@ -296,6 +301,8 @@ spec: name: docker - mountPath: /app/pythonpath name: pythonpath + - mountPath: /app/localization + name: localization - mountPath: /app/security name: security - mountPath: /app/assets @@ -307,6 +314,9 @@ spec: - name: pythonpath configMap: name: superset-pythonpath + - name: localization + configMap: + name: superset-localization - name: security configMap: name: superset-security diff --git a/tutoraspects/patches/kustomization-configmapgenerator b/tutoraspects/patches/kustomization-configmapgenerator index a7fd11e38..5272ae682 100644 --- a/tutoraspects/patches/kustomization-configmapgenerator +++ b/tutoraspects/patches/kustomization-configmapgenerator @@ -68,6 +68,13 @@ labels: app.kubernetes.io/name: superset +- name: superset-localization + files:{% for file in "aspects/apps/superset/localization"|walk_templates %} + - plugins/{{ file }}{% endfor %} + options: + labels: + app.kubernetes.io/name: superset + - name: superset-security files: - plugins/aspects/apps/superset/security/roles.json diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/locale.yaml b/tutoraspects/templates/aspects/apps/superset/localization/locale.yaml similarity index 100% rename from tutoraspects/templates/aspects/apps/superset/pythonpath/locale.yaml rename to tutoraspects/templates/aspects/apps/superset/localization/locale.yaml diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index 79a9f0543..25849361f 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -33,7 +33,7 @@ FILE_NAME_ATTRIBUTE = "_file_name" -TRANSLATIONS_FILE_PATH = "/app/pythonpath/locale.yaml" +TRANSLATIONS_FILE_PATH = "/app/localization/locale.yaml" ASSETS_FILE_PATH = "/app/pythonpath/assets.yaml" merged_data = {} diff --git a/tutoraspects/templates/base-docker-compose-services b/tutoraspects/templates/base-docker-compose-services index 9247c726c..fbd1d759c 100644 --- a/tutoraspects/templates/base-docker-compose-services +++ b/tutoraspects/templates/base-docker-compose-services @@ -3,6 +3,7 @@ image: {{ DOCKER_IMAGE_SUPERSET }} volumes: - ../../env/plugins/aspects/apps/superset/docker:/app/docker - ../../env/plugins/aspects/apps/superset/pythonpath:/app/pythonpath + - ../../env/plugins/aspects/apps/superset/localization:/app/localization - ../../env/plugins/aspects/apps/superset/security:/app/security - ../../env/plugins/aspects/apps/superset/superset_home:/app/superset_home - ../../env/plugins/aspects/apps/superset/assets:/app/assets From 770ce38b5b15e4d7829a238ea8b56be931d44278 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 10 Oct 2023 16:26:13 -0400 Subject: [PATCH 3/3] chore: Bump tutor version to work around py2neo issue, upgrade --- requirements/base.txt | 12 +++++------ requirements/dev.txt | 47 ++++++++++++++----------------------------- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5dce24800..5d54d5288 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -25,7 +25,7 @@ click==8.1.7 # tutor future==0.18.3 # via pyseeyou -google-auth==2.23.2 +google-auth==2.23.3 # via kubernetes idna==3.4 # via requests @@ -35,7 +35,7 @@ kubernetes==28.1.0 # via tutor markupsafe==2.1.3 # via jinja2 -mypy==1.5.1 +mypy==1.6.0 # via tutor mypy-extensions==1.0.0 # via mypy @@ -63,7 +63,7 @@ pyyaml==6.0.1 # via # kubernetes # tutor -regex==2023.8.8 +regex==2023.10.3 # via parsimonious requests==2.31.0 # via @@ -85,15 +85,15 @@ toolz==0.12.0 # via pyseeyou transifex-python==3.4.0 # via -r requirements/base.in -tutor==16.1.1 +tutor==16.1.3 # via -r requirements/base.in typing-extensions==4.8.0 # via # mypy # tutor -urllib3==1.26.16 +urllib3==1.26.17 # via # kubernetes # requests -websocket-client==1.6.3 +websocket-client==1.6.4 # via kubernetes diff --git a/requirements/dev.txt b/requirements/dev.txt index 65fda716f..e1c9cf2f3 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,12 +5,14 @@ # make upgrade # altgraph==0.17.4 - # via pyinstaller + # via + # macholib + # pyinstaller appdirs==1.4.4 # via # -r requirements/base.txt # tutor -astroid==2.15.8 +astroid==3.0.0 # via pylint asttokens==2.4.0 # via @@ -29,8 +31,6 @@ certifi==2023.7.22 # -r requirements/base.txt # kubernetes # requests -cffi==1.16.0 - # via cryptography charset-normalizer==3.3.0 # via # -r requirements/base.txt @@ -41,8 +41,6 @@ click==8.1.7 # black # transifex-python # tutor -cryptography==41.0.4 - # via secretstorage dill==0.3.7 # via pylint docutils==0.20.1 @@ -51,7 +49,7 @@ future==0.18.3 # via # -r requirements/base.txt # pyseeyou -google-auth==2.23.2 +google-auth==2.23.3 # via # -r requirements/base.txt # kubernetes @@ -64,16 +62,10 @@ importlib-metadata==6.8.0 # keyring # pyinstaller # twine -importlib-resources==6.1.0 - # via keyring isort==5.12.0 # via pylint jaraco-classes==3.3.0 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # -r requirements/base.txt @@ -84,8 +76,8 @@ kubernetes==28.1.0 # via # -r requirements/base.txt # tutor -lazy-object-proxy==1.9.0 - # via astroid +macholib==1.16.3 + # via pyinstaller markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 @@ -98,7 +90,7 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==10.1.0 # via jaraco-classes -mypy==1.5.1 +mypy==1.6.0 # via # -r requirements/base.txt # tutor @@ -126,7 +118,7 @@ pathspec==0.11.2 # via black pkginfo==1.9.6 # via twine -platformdirs==3.10.0 +platformdirs==3.11.0 # via # black # pylint @@ -139,8 +131,6 @@ pyasn1-modules==0.3.0 # via # -r requirements/base.txt # google-auth -pycparser==2.21 - # via cffi pycryptodome==3.19.0 # via # -r requirements/base.txt @@ -153,7 +143,7 @@ pyinstaller==6.0.0 # via -r requirements/dev.in pyinstaller-hooks-contrib==2023.9 # via pyinstaller -pylint==2.17.7 +pylint==3.0.1 # via -r requirements/dev.in pyseeyou==1.0.2 # via @@ -174,7 +164,7 @@ pyyaml==6.0.1 # tutor readme-renderer==42.0 # via twine -regex==2023.8.8 +regex==2023.10.3 # via # -r requirements/base.txt # parsimonious @@ -200,8 +190,6 @@ rsa==4.9 # via # -r requirements/base.txt # google-auth -secretstorage==3.3.3 - # via keyring six==1.16.0 # via # -r requirements/base.txt @@ -222,7 +210,7 @@ toolz==0.12.0 # pyseeyou transifex-python==3.4.0 # via -r requirements/base.txt -tutor==16.1.1 +tutor==16.1.3 # via -r requirements/base.txt twine==4.0.2 # via -r requirements/dev.in @@ -233,24 +221,19 @@ typing-extensions==4.8.0 # black # mypy # pylint - # rich # tutor -urllib3==1.26.16 +urllib3==1.26.17 # via # -r requirements/base.txt # kubernetes # requests # twine -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/base.txt # kubernetes -wrapt==1.15.0 - # via astroid zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools