diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 182e28226..8af60a37a 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -41,7 +41,7 @@ "OPENEDX_EXTRA_PIP_REQUIREMENTS", [ "openedx-event-sink-clickhouse==0.4.0", - "edx-event-routing-backends==v7.0.0", + "edx-event-routing-backends==v7.0.1", ], ), ("EVENT_SINK_CLICKHOUSE_MODELS", ["course_overviews", "user_profile"]), diff --git a/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0025_problem_responses.py b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0025_problem_responses.py new file mode 100644 index 000000000..7b0663d6e --- /dev/null +++ b/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0025_problem_responses.py @@ -0,0 +1,152 @@ +""" +Relies on ERB to do the right thing with responses instead of trying to +escape quotes for lists. +""" + +from alembic import op + +revision = "0025" +down_revision = "0024" +branch_labels = None +depends_on = None +on_cluster = ( + " ON CLUSTER '{{CLICKHOUSE_CLUSTER_NAME}}' " + if "{{CLICKHOUSE_CLUSTER_NAME}}" + else "" +) +engine = ( + "ReplicatedReplacingMergeTree" + if "{{CLICKHOUSE_CLUSTER_NAME}}" + else "ReplacingMergeTree" +) + + +TABLE_NAME = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }}" +VIEW_NAME = "{{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_TRANSFORM_MV }}" + +PROBLEM_DDL = f""" +CREATE OR REPLACE TABLE {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_PROBLEM_EVENTS_TABLE }} +{on_cluster} +( + `event_id` UUID NOT NULL, + `emission_time` DateTime NOT NULL, + `actor_id` String NOT NULL, + `object_id` String NOT NULL, + `course_key` String NOT NULL, + `org` String NOT NULL, + `verb_id` LowCardinality(String) NOT NULL, + `responses` String, + `scaled_score` String, + `success` Bool, + `interaction_type` LowCardinality(String), + `attempts` Int16 +) ENGINE = {engine} +PRIMARY KEY (org, course_key, verb_id) +ORDER BY (org, course_key, verb_id, emission_time, actor_id, object_id, responses, success, event_id); +""" + + +OLD_PROBLEM_QUERY = """ +SELECT + event_id, + cast(emission_time as DateTime) as emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + replaceAll( + replaceAll( + JSON_VALUE(event_str, '$.result.response'), + '\\\'', '\\\\\\\'' + ), '"', '\\\'') as responses, + JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, + if( + verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', + cast(JSON_VALUE(event_str, '$.result.success') as Bool), + false + ) as success, + JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, + if( + verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', + cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), + 0 + ) as attempts +FROM + {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} +WHERE + verb_id in ( + 'https://w3id.org/xapi/acrossx/verbs/evaluated', + 'http://adlnet.gov/expapi/verbs/passed', + 'http://adlnet.gov/expapi/verbs/asked' + ); +""" + + +NEW_PROBLEM_QUERY = """ +SELECT + event_id, + cast(emission_time as DateTime) as emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] AS course_key, + org, + verb_id, + JSON_VALUE(event_str, '$.result.response') as responses, + JSON_VALUE(event_str, '$.result.score.scaled') as scaled_score, + if( + verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', + cast(JSON_VALUE(event_str, '$.result.success') as Bool), + false + ) as success, + JSON_VALUE(event_str, '$.object.definition.interactionType') as interaction_type, + if( + verb_id = 'https://w3id.org/xapi/acrossx/verbs/evaluated', + cast(JSON_VALUE(event_str, '$.object.definition.extensions."http://id.tincanapi.com/extension/attempt-id"') as Int16), + 0 + ) as attempts +FROM + {{ ASPECTS_XAPI_DATABASE }}.{{ ASPECTS_XAPI_TABLE }} +WHERE + verb_id in ( + 'https://w3id.org/xapi/acrossx/verbs/evaluated', + 'http://adlnet.gov/expapi/verbs/passed', + 'http://adlnet.gov/expapi/verbs/asked' + ); +""" + + +def drop_objects(): + op.execute( + f""" + DROP TABLE IF EXISTS {TABLE_NAME} {on_cluster} + """ + ) + + op.execute( + f""" + DROP VIEW IF EXISTS {VIEW_NAME} {on_cluster} + """ + ) + + +def upgrade(): + drop_objects() + op.execute(PROBLEM_DDL) + op.execute(f"INSERT INTO {TABLE_NAME} {NEW_PROBLEM_QUERY}") + op.execute( + f""" + CREATE MATERIALIZED VIEW {VIEW_NAME} {on_cluster} TO {TABLE_NAME} AS {NEW_PROBLEM_QUERY} + """ + ) + + +def downgrade(): + drop_objects() + op.execute(PROBLEM_DDL) + op.execute(f"INSERT INTO {TABLE_NAME} {OLD_PROBLEM_QUERY}") + op.execute( + f""" + CREATE MATERIALIZED VIEW {VIEW_NAME} {on_cluster} TO {TABLE_NAME} AS {OLD_PROBLEM_QUERY} + """ + ) diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Operator_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Operator_Dashboard.yaml index c6364199f..534f35c78 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Operator_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Operator_Dashboard.yaml @@ -878,9 +878,9 @@ position: ## Dependencies: {% for requirement in OPENEDX_EXTRA_PIP_REQUIREMENTS %} {% if 'openedx-event-sink-clickhouse' in requirement %} - - **Clickhouse Event Sink Version**: {{ requirement.split('==')[1] }} \n + - **Clickhouse Event Sink Version**: {{ requirement }} \n {% elif 'event-routing-backends' in requirement %} - - **Event Routing Backends Version**: {{ requirement.split('==')[1] }} \n + - **Event Routing Backends Version**: {{ requirement }} \n {% endif %} {% endfor %} " diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index 94d12be8a..0962f17a9 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -16,10 +16,11 @@ select success, arrayJoin( if( - startsWith(responses, '['), - cast(responses as Array(String)), + JSONArrayLength(responses) > 0, + JSONExtractArrayRaw(responses), [responses] - )) as responses + ) + ) as responses from problem_responses where